2024-11-15 14:14:30,547 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 14:14:30,568 main DEBUG Took 0.014849 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 14:14:30,568 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 14:14:30,569 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 14:14:30,570 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 14:14:30,572 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 14:14:30,583 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 14:14:30,598 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,600 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 14:14:30,601 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,601 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 14:14:30,602 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,603 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 14:14:30,604 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,605 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 14:14:30,606 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,606 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 14:14:30,607 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,608 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 14:14:30,609 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,609 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 14:14:30,610 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,610 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 14:14:30,611 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,611 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 14:14:30,612 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,613 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 14:14:30,613 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,614 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 14:14:30,615 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,615 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 14:14:30,616 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,616 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 14:14:30,619 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 14:14:30,621 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 14:14:30,624 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 14:14:30,625 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 14:14:30,627 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 14:14:30,627 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 14:14:30,638 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 14:14:30,641 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 14:14:30,643 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 14:14:30,643 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 14:14:30,644 main DEBUG createAppenders(={Console}) 2024-11-15 14:14:30,645 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-15 14:14:30,645 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 14:14:30,645 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-15 14:14:30,646 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 14:14:30,646 main DEBUG OutputStream closed 2024-11-15 14:14:30,647 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 14:14:30,647 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 14:14:30,647 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-15 14:14:30,716 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 14:14:30,719 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 14:14:30,720 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 14:14:30,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 14:14:30,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 14:14:30,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 14:14:30,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 14:14:30,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 14:14:30,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 14:14:30,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 14:14:30,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 14:14:30,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 14:14:30,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 14:14:30,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 14:14:30,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 14:14:30,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 14:14:30,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 14:14:30,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 14:14:30,732 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 14:14:30,732 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-15 14:14:30,732 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 14:14:30,733 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-15T14:14:31,008 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084 2024-11-15 14:14:31,011 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 14:14:31,011 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T14:14:31,020 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-15T14:14:31,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T14:14:31,065 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/cluster_254b7fe7-4d7b-6185-a297-1a903e5d7b50, deleteOnExit=true 2024-11-15T14:14:31,066 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T14:14:31,067 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/test.cache.data in system properties and HBase conf 2024-11-15T14:14:31,068 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T14:14:31,069 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/hadoop.log.dir in system properties and HBase conf 2024-11-15T14:14:31,069 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T14:14:31,070 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T14:14:31,070 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T14:14:31,172 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T14:14:31,270 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T14:14:31,275 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T14:14:31,276 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T14:14:31,276 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T14:14:31,277 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T14:14:31,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T14:14:31,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T14:14:31,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T14:14:31,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T14:14:31,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T14:14:31,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/nfs.dump.dir in system properties and HBase conf 2024-11-15T14:14:31,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/java.io.tmpdir in system properties and HBase conf 2024-11-15T14:14:31,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T14:14:31,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T14:14:31,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T14:14:32,662 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T14:14:32,739 INFO [Time-limited test {}] log.Log(170): Logging initialized @3324ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T14:14:32,815 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T14:14:32,891 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T14:14:32,913 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T14:14:32,913 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T14:14:32,915 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T14:14:32,927 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T14:14:32,930 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/hadoop.log.dir/,AVAILABLE} 2024-11-15T14:14:32,931 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T14:14:33,220 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/java.io.tmpdir/jetty-localhost-41741-hadoop-hdfs-3_4_1-tests_jar-_-any-7111459956725976314/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T14:14:33,232 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41741} 2024-11-15T14:14:33,232 INFO [Time-limited test {}] server.Server(415): Started @3817ms 2024-11-15T14:14:33,769 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T14:14:33,777 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T14:14:33,778 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T14:14:33,778 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T14:14:33,778 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T14:14:33,779 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/hadoop.log.dir/,AVAILABLE} 2024-11-15T14:14:33,780 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T14:14:33,892 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/java.io.tmpdir/jetty-localhost-43121-hadoop-hdfs-3_4_1-tests_jar-_-any-5889631466237275936/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T14:14:33,893 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:43121} 2024-11-15T14:14:33,894 INFO [Time-limited test {}] server.Server(415): Started @4479ms 2024-11-15T14:14:33,944 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T14:14:34,716 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/cluster_254b7fe7-4d7b-6185-a297-1a903e5d7b50/data/data1/current/BP-1184182586-172.17.0.3-1731680072216/current, will proceed with Du for space computation calculation, 2024-11-15T14:14:34,716 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/cluster_254b7fe7-4d7b-6185-a297-1a903e5d7b50/data/data2/current/BP-1184182586-172.17.0.3-1731680072216/current, will proceed with Du for space computation calculation, 2024-11-15T14:14:34,741 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T14:14:34,783 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4e8f54ede25077f6 with lease ID 0x83eb44ab3531f18e: Processing first storage report for DS-23ad49e8-4c57-496c-874d-0b5377ea84ae from datanode DatanodeRegistration(127.0.0.1:45147, datanodeUuid=08c90d75-bb97-4e69-9a5b-97e84a0a674e, infoPort=43581, infoSecurePort=0, ipcPort=46791, storageInfo=lv=-57;cid=testClusterID;nsid=746472713;c=1731680072216) 2024-11-15T14:14:34,784 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4e8f54ede25077f6 with lease ID 0x83eb44ab3531f18e: from storage DS-23ad49e8-4c57-496c-874d-0b5377ea84ae node DatanodeRegistration(127.0.0.1:45147, datanodeUuid=08c90d75-bb97-4e69-9a5b-97e84a0a674e, infoPort=43581, infoSecurePort=0, ipcPort=46791, storageInfo=lv=-57;cid=testClusterID;nsid=746472713;c=1731680072216), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T14:14:34,784 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4e8f54ede25077f6 with lease ID 0x83eb44ab3531f18e: Processing first storage report for DS-71c04024-0235-4089-81a5-bc8314f7a31a from datanode DatanodeRegistration(127.0.0.1:45147, datanodeUuid=08c90d75-bb97-4e69-9a5b-97e84a0a674e, infoPort=43581, infoSecurePort=0, ipcPort=46791, storageInfo=lv=-57;cid=testClusterID;nsid=746472713;c=1731680072216) 2024-11-15T14:14:34,785 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4e8f54ede25077f6 with lease ID 0x83eb44ab3531f18e: from storage DS-71c04024-0235-4089-81a5-bc8314f7a31a node DatanodeRegistration(127.0.0.1:45147, datanodeUuid=08c90d75-bb97-4e69-9a5b-97e84a0a674e, infoPort=43581, infoSecurePort=0, ipcPort=46791, storageInfo=lv=-57;cid=testClusterID;nsid=746472713;c=1731680072216), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T14:14:34,815 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084 2024-11-15T14:14:34,881 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/cluster_254b7fe7-4d7b-6185-a297-1a903e5d7b50/zookeeper_0, clientPort=49996, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/cluster_254b7fe7-4d7b-6185-a297-1a903e5d7b50/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/cluster_254b7fe7-4d7b-6185-a297-1a903e5d7b50/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T14:14:34,891 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49996 2024-11-15T14:14:34,901 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T14:14:34,904 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T14:14:35,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741825_1001 (size=7) 2024-11-15T14:14:35,535 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931 with version=8 2024-11-15T14:14:35,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/hbase-staging 2024-11-15T14:14:35,689 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T14:14:36,131 INFO [Time-limited test {}] client.ConnectionUtils(128): master/26754e923c4a:0 server-side Connection retries=45 2024-11-15T14:14:36,146 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T14:14:36,148 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T14:14:36,157 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T14:14:36,157 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T14:14:36,159 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T14:14:36,444 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T14:14:36,583 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T14:14:36,610 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T14:14:36,622 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T14:14:36,679 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8214 (auto-detected) 2024-11-15T14:14:36,683 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-15T14:14:36,729 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35807 2024-11-15T14:14:36,798 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35807 connecting to ZooKeeper ensemble=127.0.0.1:49996 2024-11-15T14:14:37,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:358070x0, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T14:14:37,037 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35807-0x1014006c21b0000 connected 2024-11-15T14:14:37,215 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T14:14:37,224 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T14:14:37,256 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T14:14:37,262 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931, hbase.cluster.distributed=false 2024-11-15T14:14:37,307 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T14:14:37,339 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35807 2024-11-15T14:14:37,343 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35807 2024-11-15T14:14:37,359 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35807 2024-11-15T14:14:37,370 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35807 2024-11-15T14:14:37,370 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35807 2024-11-15T14:14:37,615 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/26754e923c4a:0 server-side Connection retries=45 2024-11-15T14:14:37,617 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T14:14:37,617 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T14:14:37,618 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T14:14:37,618 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T14:14:37,618 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T14:14:37,621 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T14:14:37,625 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T14:14:37,630 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40177 2024-11-15T14:14:37,633 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40177 connecting to ZooKeeper ensemble=127.0.0.1:49996 2024-11-15T14:14:37,636 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T14:14:37,656 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T14:14:37,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401770x0, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T14:14:37,707 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:401770x0, quorum=127.0.0.1:49996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T14:14:37,712 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T14:14:37,719 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40177-0x1014006c21b0001 connected 2024-11-15T14:14:37,736 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T14:14:37,740 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T14:14:37,752 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T14:14:37,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40177 2024-11-15T14:14:37,778 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40177 2024-11-15T14:14:37,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40177 2024-11-15T14:14:37,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40177 2024-11-15T14:14:37,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40177 2024-11-15T14:14:37,842 DEBUG [M:0;26754e923c4a:35807 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;26754e923c4a:35807 2024-11-15T14:14:37,849 INFO [master/26754e923c4a:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/26754e923c4a,35807,1731680075796 2024-11-15T14:14:37,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T14:14:37,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T14:14:37,871 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/26754e923c4a,35807,1731680075796 2024-11-15T14:14:37,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T14:14:37,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:14:37,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:14:37,919 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T14:14:37,923 INFO [master/26754e923c4a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/26754e923c4a,35807,1731680075796 from backup master directory 2024-11-15T14:14:37,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T14:14:37,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/26754e923c4a,35807,1731680075796 2024-11-15T14:14:37,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T14:14:37,935 WARN [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T14:14:37,936 INFO [master/26754e923c4a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=26754e923c4a,35807,1731680075796 2024-11-15T14:14:37,945 INFO [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T14:14:37,949 INFO [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T14:14:38,088 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/hbase.id] with ID: f9dcf31d-c076-4b78-aeaf-342511b996a6 2024-11-15T14:14:38,088 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/.tmp/hbase.id 2024-11-15T14:14:38,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741826_1002 (size=42) 2024-11-15T14:14:38,131 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/.tmp/hbase.id]:[hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/hbase.id] 2024-11-15T14:14:38,230 INFO [master/26754e923c4a:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T14:14:38,238 INFO [master/26754e923c4a:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T14:14:38,286 INFO [master/26754e923c4a:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 39ms. 2024-11-15T14:14:38,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:14:38,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:14:38,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741827_1003 (size=196) 2024-11-15T14:14:38,410 INFO [master/26754e923c4a:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T14:14:38,415 INFO [master/26754e923c4a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T14:14:38,439 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:38,446 INFO [master/26754e923c4a:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T14:14:38,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741828_1004 (size=1189) 2024-11-15T14:14:38,949 INFO [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store 2024-11-15T14:14:38,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741829_1005 (size=34) 2024-11-15T14:14:39,003 INFO [master/26754e923c4a:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T14:14:39,009 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:14:39,011 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T14:14:39,012 INFO [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T14:14:39,012 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T14:14:39,014 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T14:14:39,015 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T14:14:39,015 INFO [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T14:14:39,018 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731680079011Disabling compacts and flushes for region at 1731680079011Disabling writes for close at 1731680079015 (+4 ms)Writing region close event to WAL at 1731680079015Closed at 1731680079015 2024-11-15T14:14:39,028 WARN [master/26754e923c4a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/.initializing 2024-11-15T14:14:39,028 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/WALs/26754e923c4a,35807,1731680075796 2024-11-15T14:14:39,048 INFO [master/26754e923c4a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T14:14:39,080 INFO [master/26754e923c4a:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=26754e923c4a%2C35807%2C1731680075796, suffix=, logDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/WALs/26754e923c4a,35807,1731680075796, archiveDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/oldWALs, maxLogs=10 2024-11-15T14:14:39,118 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/WALs/26754e923c4a,35807,1731680075796/26754e923c4a%2C35807%2C1731680075796.1731680079090, exclude list is [], retry=0 2024-11-15T14:14:39,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45147,DS-23ad49e8-4c57-496c-874d-0b5377ea84ae,DISK] 2024-11-15T14:14:39,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T14:14:39,219 INFO [master/26754e923c4a:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/WALs/26754e923c4a,35807,1731680075796/26754e923c4a%2C35807%2C1731680075796.1731680079090 2024-11-15T14:14:39,221 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43581:43581)] 2024-11-15T14:14:39,221 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T14:14:39,222 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:14:39,227 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T14:14:39,228 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T14:14:39,282 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T14:14:39,325 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T14:14:39,330 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:39,334 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T14:14:39,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T14:14:39,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T14:14:39,341 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:39,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:14:39,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T14:14:39,351 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T14:14:39,351 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:39,353 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:14:39,354 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T14:14:39,360 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T14:14:39,360 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:39,362 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:14:39,363 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T14:14:39,368 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T14:14:39,370 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T14:14:39,383 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T14:14:39,385 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T14:14:39,397 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T14:14:39,407 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T14:14:39,425 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T14:14:39,428 INFO [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62244546, jitterRate=-0.07248398661613464}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T14:14:39,448 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731680079247Initializing all the Stores at 1731680079250 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731680079251 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680079252 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680079253 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680079253Cleaning up temporary data from old regions at 1731680079385 (+132 ms)Region opened successfully at 1731680079447 (+62 ms) 2024-11-15T14:14:39,453 INFO [master/26754e923c4a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T14:14:39,496 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3846c39d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=26754e923c4a/172.17.0.3:0 2024-11-15T14:14:39,540 INFO [master/26754e923c4a:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T14:14:39,556 INFO [master/26754e923c4a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T14:14:39,557 INFO [master/26754e923c4a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T14:14:39,563 INFO [master/26754e923c4a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T14:14:39,565 INFO [master/26754e923c4a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-15T14:14:39,573 INFO [master/26754e923c4a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-15T14:14:39,573 INFO [master/26754e923c4a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T14:14:39,613 INFO [master/26754e923c4a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T14:14:39,633 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T14:14:39,697 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T14:14:39,700 INFO [master/26754e923c4a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T14:14:39,705 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T14:14:39,716 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T14:14:39,720 INFO [master/26754e923c4a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T14:14:39,725 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T14:14:39,733 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T14:14:39,735 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T14:14:39,741 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T14:14:39,787 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T14:14:39,799 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T14:14:39,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T14:14:39,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T14:14:39,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:14:39,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:14:39,822 INFO [master/26754e923c4a:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=26754e923c4a,35807,1731680075796, sessionid=0x1014006c21b0000, setting cluster-up flag (Was=false) 2024-11-15T14:14:39,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:14:39,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:14:39,891 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T14:14:39,893 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=26754e923c4a,35807,1731680075796 2024-11-15T14:14:39,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:14:39,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:14:39,950 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T14:14:39,957 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=26754e923c4a,35807,1731680075796 2024-11-15T14:14:39,977 INFO [master/26754e923c4a:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T14:14:40,031 INFO [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(746): ClusterId : f9dcf31d-c076-4b78-aeaf-342511b996a6 2024-11-15T14:14:40,037 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T14:14:40,055 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T14:14:40,055 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T14:14:40,068 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T14:14:40,069 DEBUG [RS:0;26754e923c4a:40177 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1ce737, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=26754e923c4a/172.17.0.3:0 2024-11-15T14:14:40,112 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T14:14:40,114 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;26754e923c4a:40177 2024-11-15T14:14:40,121 INFO [RS:0;26754e923c4a:40177 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T14:14:40,121 INFO [RS:0;26754e923c4a:40177 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T14:14:40,122 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T14:14:40,126 INFO [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(2659): reportForDuty to master=26754e923c4a,35807,1731680075796 with port=40177, startcode=1731680077530 2024-11-15T14:14:40,138 INFO [master/26754e923c4a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T14:14:40,152 INFO [master/26754e923c4a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T14:14:40,154 DEBUG [RS:0;26754e923c4a:40177 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T14:14:40,161 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 26754e923c4a,35807,1731680075796 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T14:14:40,177 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/26754e923c4a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T14:14:40,178 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/26754e923c4a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T14:14:40,178 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/26754e923c4a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T14:14:40,179 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/26754e923c4a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T14:14:40,180 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/26754e923c4a:0, corePoolSize=10, maxPoolSize=10 2024-11-15T14:14:40,181 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/26754e923c4a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T14:14:40,181 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/26754e923c4a:0, corePoolSize=2, maxPoolSize=2 2024-11-15T14:14:40,181 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/26754e923c4a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T14:14:40,195 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T14:14:40,196 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T14:14:40,199 INFO [master/26754e923c4a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731680110199 2024-11-15T14:14:40,203 INFO [master/26754e923c4a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T14:14:40,204 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35273, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T14:14:40,204 INFO [master/26754e923c4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T14:14:40,207 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:40,208 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T14:14:40,209 INFO [master/26754e923c4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T14:14:40,210 INFO [master/26754e923c4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T14:14:40,210 INFO [master/26754e923c4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T14:14:40,211 INFO [master/26754e923c4a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T14:14:40,215 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,218 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:40,223 INFO [master/26754e923c4a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T14:14:40,224 INFO [master/26754e923c4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T14:14:40,225 INFO [master/26754e923c4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T14:14:40,228 INFO [master/26754e923c4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T14:14:40,229 INFO [master/26754e923c4a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T14:14:40,232 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/26754e923c4a:0:becomeActiveMaster-HFileCleaner.large.0-1731680080231,5,FailOnTimeoutGroup] 2024-11-15T14:14:40,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741831_1007 (size=1321) 2024-11-15T14:14:40,235 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/26754e923c4a:0:becomeActiveMaster-HFileCleaner.small.0-1731680080232,5,FailOnTimeoutGroup] 2024-11-15T14:14:40,235 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,235 INFO [master/26754e923c4a:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T14:14:40,237 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,237 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,238 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T14:14:40,239 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931 2024-11-15T14:14:40,249 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-15T14:14:40,249 WARN [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-15T14:14:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741832_1008 (size=32) 2024-11-15T14:14:40,351 INFO [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(2659): reportForDuty to master=26754e923c4a,35807,1731680075796 with port=40177, startcode=1731680077530 2024-11-15T14:14:40,353 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35807 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 26754e923c4a,40177,1731680077530 2024-11-15T14:14:40,356 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35807 {}] master.ServerManager(517): Registering regionserver=26754e923c4a,40177,1731680077530 2024-11-15T14:14:40,366 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931 2024-11-15T14:14:40,366 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39863 2024-11-15T14:14:40,366 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T14:14:40,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T14:14:40,410 DEBUG [RS:0;26754e923c4a:40177 {}] zookeeper.ZKUtil(111): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/26754e923c4a,40177,1731680077530 2024-11-15T14:14:40,410 WARN [RS:0;26754e923c4a:40177 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T14:14:40,410 INFO [RS:0;26754e923c4a:40177 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T14:14:40,411 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/WALs/26754e923c4a,40177,1731680077530 2024-11-15T14:14:40,421 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [26754e923c4a,40177,1731680077530] 2024-11-15T14:14:40,451 INFO [RS:0;26754e923c4a:40177 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T14:14:40,473 INFO [RS:0;26754e923c4a:40177 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T14:14:40,479 INFO [RS:0;26754e923c4a:40177 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T14:14:40,480 INFO [RS:0;26754e923c4a:40177 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,481 INFO [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T14:14:40,488 INFO [RS:0;26754e923c4a:40177 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T14:14:40,489 INFO [RS:0;26754e923c4a:40177 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,490 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/26754e923c4a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T14:14:40,490 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/26754e923c4a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T14:14:40,490 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/26754e923c4a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T14:14:40,490 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/26754e923c4a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T14:14:40,490 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/26754e923c4a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T14:14:40,491 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/26754e923c4a:0, corePoolSize=2, maxPoolSize=2 2024-11-15T14:14:40,491 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T14:14:40,491 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/26754e923c4a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T14:14:40,491 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/26754e923c4a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T14:14:40,492 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/26754e923c4a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T14:14:40,492 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/26754e923c4a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T14:14:40,492 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/26754e923c4a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T14:14:40,492 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/26754e923c4a:0, corePoolSize=3, maxPoolSize=3 2024-11-15T14:14:40,492 DEBUG [RS:0;26754e923c4a:40177 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0, corePoolSize=3, maxPoolSize=3 2024-11-15T14:14:40,494 INFO [RS:0;26754e923c4a:40177 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,494 INFO [RS:0;26754e923c4a:40177 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,494 INFO [RS:0;26754e923c4a:40177 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,495 INFO [RS:0;26754e923c4a:40177 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,495 INFO [RS:0;26754e923c4a:40177 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,495 INFO [RS:0;26754e923c4a:40177 {}] hbase.ChoreService(168): Chore ScheduledChore name=26754e923c4a,40177,1731680077530-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T14:14:40,523 INFO [RS:0;26754e923c4a:40177 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T14:14:40,526 INFO [RS:0;26754e923c4a:40177 {}] hbase.ChoreService(168): Chore ScheduledChore name=26754e923c4a,40177,1731680077530-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,526 INFO [RS:0;26754e923c4a:40177 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,526 INFO [RS:0;26754e923c4a:40177 {}] regionserver.Replication(171): 26754e923c4a,40177,1731680077530 started 2024-11-15T14:14:40,552 INFO [RS:0;26754e923c4a:40177 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:40,552 INFO [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1482): Serving as 26754e923c4a,40177,1731680077530, RpcServer on 26754e923c4a/172.17.0.3:40177, sessionid=0x1014006c21b0001 2024-11-15T14:14:40,554 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T14:14:40,554 DEBUG [RS:0;26754e923c4a:40177 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 26754e923c4a,40177,1731680077530 2024-11-15T14:14:40,555 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26754e923c4a,40177,1731680077530' 2024-11-15T14:14:40,555 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T14:14:40,557 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T14:14:40,559 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T14:14:40,559 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T14:14:40,559 DEBUG [RS:0;26754e923c4a:40177 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 26754e923c4a,40177,1731680077530 2024-11-15T14:14:40,560 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26754e923c4a,40177,1731680077530' 2024-11-15T14:14:40,560 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T14:14:40,561 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T14:14:40,562 DEBUG [RS:0;26754e923c4a:40177 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T14:14:40,562 INFO [RS:0;26754e923c4a:40177 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T14:14:40,562 INFO [RS:0;26754e923c4a:40177 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T14:14:40,664 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:14:40,674 INFO [RS:0;26754e923c4a:40177 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T14:14:40,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T14:14:40,679 INFO [RS:0;26754e923c4a:40177 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=26754e923c4a%2C40177%2C1731680077530, suffix=, logDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/WALs/26754e923c4a,40177,1731680077530, archiveDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/oldWALs, maxLogs=32 2024-11-15T14:14:40,683 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T14:14:40,684 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:40,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T14:14:40,686 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T14:14:40,691 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T14:14:40,691 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:40,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T14:14:40,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T14:14:40,700 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T14:14:40,700 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:40,701 DEBUG [RS:0;26754e923c4a:40177 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/WALs/26754e923c4a,40177,1731680077530/26754e923c4a%2C40177%2C1731680077530.1731680080683, exclude list is [], retry=0 2024-11-15T14:14:40,701 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T14:14:40,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T14:14:40,705 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T14:14:40,705 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:40,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T14:14:40,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45147,DS-23ad49e8-4c57-496c-874d-0b5377ea84ae,DISK] 2024-11-15T14:14:40,707 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T14:14:40,709 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740 2024-11-15T14:14:40,710 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740 2024-11-15T14:14:40,716 INFO [RS:0;26754e923c4a:40177 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/WALs/26754e923c4a,40177,1731680077530/26754e923c4a%2C40177%2C1731680077530.1731680080683 2024-11-15T14:14:40,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T14:14:40,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T14:14:40,719 DEBUG [RS:0;26754e923c4a:40177 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43581:43581)] 2024-11-15T14:14:40,719 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T14:14:40,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T14:14:40,749 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T14:14:40,750 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71493648, jitterRate=0.0653383731842041}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T14:14:40,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731680080664Initializing all the Stores at 1731680080670 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731680080670Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731680080677 (+7 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680080677Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731680080677Cleaning up temporary data from old regions at 1731680080718 (+41 ms)Region opened successfully at 1731680080754 (+36 ms) 2024-11-15T14:14:40,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T14:14:40,759 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T14:14:40,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T14:14:40,760 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T14:14:40,760 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T14:14:40,765 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T14:14:40,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731680080758Disabling compacts and flushes for region at 1731680080758Disabling writes for close at 1731680080760 (+2 ms)Writing region close event to WAL at 1731680080764 (+4 ms)Closed at 1731680080765 (+1 ms) 2024-11-15T14:14:40,771 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T14:14:40,771 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T14:14:40,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T14:14:40,793 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T14:14:40,801 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T14:14:40,954 DEBUG [26754e923c4a:35807 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T14:14:40,971 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:14:40,979 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 26754e923c4a,40177,1731680077530, state=OPENING 2024-11-15T14:14:41,016 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T14:14:41,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:14:41,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:14:41,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T14:14:41,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T14:14:41,027 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T14:14:41,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:14:41,214 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T14:14:41,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49909, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T14:14:41,245 INFO [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T14:14:41,245 INFO [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T14:14:41,249 INFO [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T14:14:41,254 INFO [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=26754e923c4a%2C40177%2C1731680077530.meta, suffix=.meta, logDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/WALs/26754e923c4a,40177,1731680077530, archiveDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/oldWALs, maxLogs=32 2024-11-15T14:14:41,280 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/WALs/26754e923c4a,40177,1731680077530/26754e923c4a%2C40177%2C1731680077530.meta.1731680081257.meta, exclude list is [], retry=0 2024-11-15T14:14:41,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45147,DS-23ad49e8-4c57-496c-874d-0b5377ea84ae,DISK] 2024-11-15T14:14:41,300 INFO [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/WALs/26754e923c4a,40177,1731680077530/26754e923c4a%2C40177%2C1731680077530.meta.1731680081257.meta 2024-11-15T14:14:41,307 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43581:43581)] 2024-11-15T14:14:41,308 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T14:14:41,310 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T14:14:41,315 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T14:14:41,324 INFO [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T14:14:41,332 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T14:14:41,333 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:14:41,334 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T14:14:41,334 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T14:14:41,347 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T14:14:41,350 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T14:14:41,350 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:41,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T14:14:41,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T14:14:41,356 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T14:14:41,357 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:41,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T14:14:41,362 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T14:14:41,365 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T14:14:41,365 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:41,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T14:14:41,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T14:14:41,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T14:14:41,371 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:41,375 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T14:14:41,375 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T14:14:41,378 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740 2024-11-15T14:14:41,382 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740 2024-11-15T14:14:41,388 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T14:14:41,388 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T14:14:41,390 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T14:14:41,397 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T14:14:41,405 INFO [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74867904, jitterRate=0.11561870574951172}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T14:14:41,406 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T14:14:41,410 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731680081335Writing region info on filesystem at 1731680081336 (+1 ms)Initializing all the Stores at 1731680081342 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731680081342Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731680081346 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680081347 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731680081347Cleaning up temporary data from old regions at 1731680081388 (+41 ms)Running coprocessor post-open hooks at 1731680081407 (+19 ms)Region opened successfully at 1731680081410 (+3 ms) 2024-11-15T14:14:41,421 INFO [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731680081202 2024-11-15T14:14:41,435 DEBUG [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T14:14:41,436 INFO [RS_OPEN_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T14:14:41,440 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:14:41,443 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 26754e923c4a,40177,1731680077530, state=OPEN 2024-11-15T14:14:41,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T14:14:41,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T14:14:41,496 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T14:14:41,496 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T14:14:41,496 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:41,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T14:14:41,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=26754e923c4a,40177,1731680077530 in 467 msec 2024-11-15T14:14:41,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T14:14:41,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 726 msec 2024-11-15T14:14:41,515 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T14:14:41,515 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T14:14:41,539 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:14:41,541 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:14:41,565 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:14:41,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60637, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:14:41,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5680 sec 2024-11-15T14:14:41,595 INFO [master/26754e923c4a:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731680081595, completionTime=-1 2024-11-15T14:14:41,598 INFO [master/26754e923c4a:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-15T14:14:41,599 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T14:14:41,637 INFO [master/26754e923c4a:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-15T14:14:41,637 INFO [master/26754e923c4a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731680141637 2024-11-15T14:14:41,637 INFO [master/26754e923c4a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731680201637 2024-11-15T14:14:41,637 INFO [master/26754e923c4a:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 38 msec 2024-11-15T14:14:41,640 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26754e923c4a,35807,1731680075796-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:41,641 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26754e923c4a,35807,1731680075796-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:41,641 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26754e923c4a,35807,1731680075796-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:41,643 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-26754e923c4a:35807, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:41,644 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:41,648 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:41,654 DEBUG [master/26754e923c4a:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T14:14:41,700 INFO [master/26754e923c4a:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.764sec 2024-11-15T14:14:41,702 INFO [master/26754e923c4a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T14:14:41,704 INFO [master/26754e923c4a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T14:14:41,705 INFO [master/26754e923c4a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T14:14:41,706 INFO [master/26754e923c4a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T14:14:41,706 INFO [master/26754e923c4a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T14:14:41,707 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26754e923c4a,35807,1731680075796-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T14:14:41,708 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26754e923c4a,35807,1731680075796-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T14:14:41,725 DEBUG [master/26754e923c4a:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T14:14:41,726 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T14:14:41,727 INFO [master/26754e923c4a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26754e923c4a,35807,1731680075796-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T14:14:41,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:41,750 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T14:14:41,750 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T14:14:41,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:14:41,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:14:41,775 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:14:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:14:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:14:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5db9d8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:14:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:14:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:14:41,806 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:14:41,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:41,817 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:14:41,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:14:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:14:41,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:14:41,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=26754e923c4a,35807,1731680075796 2024-11-15T14:14:41,897 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=620, ProcessCount=11, AvailableMemoryMB=524 2024-11-15T14:14:41,937 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T14:14:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 26754e923c4a,35807,1731680075796 2024-11-15T14:14:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b65baa9 2024-11-15T14:14:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T14:14:41,954 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T14:14:41,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T14:14:41,963 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T14:14:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T14:14:41,980 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T14:14:41,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-15T14:14:41,993 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T14:14:42,009 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T14:14:42,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741835_1011 (size=1061) 2024-11-15T14:14:42,071 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931 2024-11-15T14:14:42,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741836_1012 (size=53) 2024-11-15T14:14:42,111 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:14:42,111 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 785aa89faccd90f1bafb73964185be68, disabling compactions & flushes 2024-11-15T14:14:42,111 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:42,112 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:42,112 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. after waiting 0 ms 2024-11-15T14:14:42,112 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:42,112 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:42,112 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 785aa89faccd90f1bafb73964185be68: Waiting for close lock at 1731680082111Disabling compacts and flushes for region at 1731680082111Disabling writes for close at 1731680082112 (+1 ms)Writing region close event to WAL at 1731680082112Closed at 1731680082112 2024-11-15T14:14:42,116 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T14:14:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T14:14:42,126 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731680082116"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731680082116"}]},"ts":"1731680082116"} 2024-11-15T14:14:42,132 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T14:14:42,135 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T14:14:42,139 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680082136"}]},"ts":"1731680082136"} 2024-11-15T14:14:42,145 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T14:14:42,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=785aa89faccd90f1bafb73964185be68, ASSIGN}] 2024-11-15T14:14:42,161 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=785aa89faccd90f1bafb73964185be68, ASSIGN 2024-11-15T14:14:42,164 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=785aa89faccd90f1bafb73964185be68, ASSIGN; state=OFFLINE, location=26754e923c4a,40177,1731680077530; forceNewPlan=false, retain=false 2024-11-15T14:14:42,317 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=785aa89faccd90f1bafb73964185be68, regionState=OPENING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:14:42,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=785aa89faccd90f1bafb73964185be68, ASSIGN because future has completed 2024-11-15T14:14:42,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:14:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T14:14:42,489 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:42,490 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} 2024-11-15T14:14:42,491 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:42,491 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:14:42,492 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:42,492 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:42,499 INFO [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:42,509 INFO [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:14:42,509 INFO [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 785aa89faccd90f1bafb73964185be68 columnFamilyName A 2024-11-15T14:14:42,510 DEBUG [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:42,513 INFO [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] regionserver.HStore(327): Store=785aa89faccd90f1bafb73964185be68/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:14:42,513 INFO [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:42,516 INFO [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:14:42,517 INFO [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 785aa89faccd90f1bafb73964185be68 columnFamilyName B 2024-11-15T14:14:42,517 DEBUG [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:42,520 INFO [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] regionserver.HStore(327): Store=785aa89faccd90f1bafb73964185be68/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:14:42,520 INFO [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:42,525 INFO [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:14:42,526 INFO [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 785aa89faccd90f1bafb73964185be68 columnFamilyName C 2024-11-15T14:14:42,526 DEBUG [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:14:42,528 INFO [StoreOpener-785aa89faccd90f1bafb73964185be68-1 {}] regionserver.HStore(327): Store=785aa89faccd90f1bafb73964185be68/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:14:42,530 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:42,530 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:42,533 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:42,534 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:42,537 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:42,538 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:42,541 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T14:14:42,545 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:42,564 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T14:14:42,567 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 785aa89faccd90f1bafb73964185be68; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70691683, jitterRate=0.053388163447380066}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T14:14:42,567 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:42,570 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 785aa89faccd90f1bafb73964185be68: Running coprocessor pre-open hook at 1731680082492Writing region info on filesystem at 1731680082492Initializing all the Stores at 1731680082496 (+4 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680082496Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680082498 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680082499 (+1 ms)Cleaning up temporary data from old regions at 1731680082538 (+39 ms)Running coprocessor post-open hooks at 1731680082567 (+29 ms)Region opened successfully at 1731680082570 (+3 ms) 2024-11-15T14:14:42,580 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., pid=6, masterSystemTime=1731680082480 2024-11-15T14:14:42,587 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:42,588 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:42,590 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=785aa89faccd90f1bafb73964185be68, regionState=OPEN, openSeqNum=2, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:14:42,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:14:42,612 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T14:14:42,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 in 280 msec 2024-11-15T14:14:42,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T14:14:42,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=785aa89faccd90f1bafb73964185be68, ASSIGN in 457 msec 2024-11-15T14:14:42,629 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T14:14:42,630 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680082629"}]},"ts":"1731680082629"} 2024-11-15T14:14:42,635 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T14:14:42,641 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T14:14:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T14:14:42,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 674 msec 2024-11-15T14:14:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T14:14:43,156 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:14:43,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,167 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:14:43,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:14:43,179 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:14:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:14:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:14:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d417a52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:14:43,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:14:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:14:43,206 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:14:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9a777e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,213 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:14:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:14:43,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:14:43,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:14:43,243 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T14:14:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 26754e923c4a,35807,1731680075796 2024-11-15T14:14:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@144e408 2024-11-15T14:14:43,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T14:14:43,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T14:14:43,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:14:43,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:14:43,281 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:14:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:14:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:14:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175526a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:14:43,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:14:43,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:14:43,313 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:14:43,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:14:43,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:14:43,333 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:14:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:14:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:14:43,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ccb4eda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:14:43,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:14:43,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:14:43,362 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:14:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc8436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:14:43,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:14:43,374 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:14:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:14:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:14:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:14:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:14:43,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:14:43,404 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:14:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:14:43,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:14:43,430 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:14:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:14:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:14:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a27ead7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:14:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:14:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:14:43,437 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:14:43,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41340957, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:14:43,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:14:43,443 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:14:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:14:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:14:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a6bf59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:14:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:14:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:14:43,448 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:14:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c50a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:14:43,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:14:43,454 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:14:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:14:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:14:43,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af5f6a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:14:43,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:14:43,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:14:43,459 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:14:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4855f112, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,463 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:14:43,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:14:43,465 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:14:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:14:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:14:43,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44bee5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:14:43,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:14:43,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:14:43,468 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:14:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a98f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,474 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:14:43,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:14:43,476 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:14:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:14:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:14:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f232f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:14:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:14:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:14:43,480 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:14:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be3f202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:14:43,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:14:43,485 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:14:43,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:14:43,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:14:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f437cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:14:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:14:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:14:43,489 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:14:43,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a903853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:14:43,499 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:14:43,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:14:43,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:14:43,502 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:14:43,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:14:43,505 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:14:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:14:43,506 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:14:43,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:14:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:14:43,509 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:14:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:14:43,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:14:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:14:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:14:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:14:43,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:14:43,519 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:14:43,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:14:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:14:43,521 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:14:43,521 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:14:43,521 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:14:43,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:14:43,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:14:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:14:43,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:14:43,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:14:43,535 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:14:43,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:14:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-15T14:14:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:14:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T14:14:43,545 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:14:43,545 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:14:43,547 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:14:43,547 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:14:43,548 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:14:43,549 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:14:43,551 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:14:43,553 DEBUG [Thread-145 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:14:43,553 DEBUG [Thread-145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:14:43,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:14:43,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:14:43,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:14:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:14:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:14:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:14:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:14:43,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:14:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:14:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:43,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T14:14:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T14:14:43,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:43,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:43,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:43,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:43,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:43,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:43,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:43,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:43,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:43,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:43,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:43,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:43,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680143813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T14:14:43,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:43,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:43,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:43,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680143819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:43,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680143827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:43,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680143858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:43,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/3388db958ff6440c8f0809727efcad5c is 50, key is test_row_0/A:col10/1731680083627/Put/seqid=0 2024-11-15T14:14:43,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731680083779, max timestamp=1731680083854Num uniques 859; 2024-11-15T14:14:43,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731680083779, max timestamp=1731680083854Num uniques 859; 2024-11-15T14:14:43,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T14:14:43,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T14:14:43,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731680083779, max timestamp=1731680083854Num uniques 859; 2024-11-15T14:14:43,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T14:14:43,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:43,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741837_1013 (size=12001) 2024-11-15T14:14:43,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:43,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:43,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680144072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:44,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680144100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680144111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680144112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680144099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:44,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:44,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:44,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T14:14:44,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680144223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680144244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680144244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680144244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680144253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:44,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:44,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:44,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:44,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/3388db958ff6440c8f0809727efcad5c 2024-11-15T14:14:44,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:44,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:44,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:44,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680144463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680144464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680144443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680144472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680144483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/d51c9ea99b5c49f9ad521055e7433ff2 is 50, key is test_row_0/B:col10/1731680083627/Put/seqid=0 2024-11-15T14:14:44,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741838_1014 (size=12001) 2024-11-15T14:14:44,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:44,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:44,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:44,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:44,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/d51c9ea99b5c49f9ad521055e7433ff2 2024-11-15T14:14:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T14:14:44,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/dc92041cfbfe4d058759c73642cafa5e is 50, key is test_row_0/C:col10/1731680083627/Put/seqid=0 2024-11-15T14:14:44,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741839_1015 (size=12001) 2024-11-15T14:14:44,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:44,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:44,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:44,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:44,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680144784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680144784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680144803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680144793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680144813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:44,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:44,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:44,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:44,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:45,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:45,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/dc92041cfbfe4d058759c73642cafa5e 2024-11-15T14:14:45,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/3388db958ff6440c8f0809727efcad5c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/3388db958ff6440c8f0809727efcad5c 2024-11-15T14:14:45,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/3388db958ff6440c8f0809727efcad5c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T14:14:45,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/d51c9ea99b5c49f9ad521055e7433ff2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d51c9ea99b5c49f9ad521055e7433ff2 2024-11-15T14:14:45,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d51c9ea99b5c49f9ad521055e7433ff2, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T14:14:45,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/dc92041cfbfe4d058759c73642cafa5e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/dc92041cfbfe4d058759c73642cafa5e 2024-11-15T14:14:45,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:45,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:45,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:45,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:45,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/dc92041cfbfe4d058759c73642cafa5e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T14:14:45,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 785aa89faccd90f1bafb73964185be68 in 1621ms, sequenceid=13, compaction requested=false 2024-11-15T14:14:45,273 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T14:14:45,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:45,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-15T14:14:45,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:45,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:14:45,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:45,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:14:45,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:45,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:14:45,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:45,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:45,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:45,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:45,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/f5e8a24474f14395944f2895ed7d22f8 is 50, key is test_row_0/A:col10/1731680085338/Put/seqid=0 2024-11-15T14:14:45,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680145422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741840_1016 (size=16681) 2024-11-15T14:14:45,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680145442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:45,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:45,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:45,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:45,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680145459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680145483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680145484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680145593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680145653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T14:14:45,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:45,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:45,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:45,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:45,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680145704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680145704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680145713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680145843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680145882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:45,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:45,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:45,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:45,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:45,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/f5e8a24474f14395944f2895ed7d22f8 2024-11-15T14:14:45,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/0be0ffb4d10e4ccaadce5c48cd8a951b is 50, key is test_row_0/B:col10/1731680085338/Put/seqid=0 2024-11-15T14:14:45,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741841_1017 (size=12001) 2024-11-15T14:14:45,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680145953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680145953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680145963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:45,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/0be0ffb4d10e4ccaadce5c48cd8a951b 2024-11-15T14:14:46,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/bd1e0cbe249b44779d2d1eb02fd8f9fd is 50, key is test_row_0/C:col10/1731680085338/Put/seqid=0 2024-11-15T14:14:46,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:46,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:46,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:46,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:46,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741842_1018 (size=12001) 2024-11-15T14:14:46,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/bd1e0cbe249b44779d2d1eb02fd8f9fd 2024-11-15T14:14:46,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/f5e8a24474f14395944f2895ed7d22f8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f5e8a24474f14395944f2895ed7d22f8 2024-11-15T14:14:46,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f5e8a24474f14395944f2895ed7d22f8, entries=250, sequenceid=48, filesize=16.3 K 2024-11-15T14:14:46,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/0be0ffb4d10e4ccaadce5c48cd8a951b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0be0ffb4d10e4ccaadce5c48cd8a951b 2024-11-15T14:14:46,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0be0ffb4d10e4ccaadce5c48cd8a951b, entries=150, sequenceid=48, filesize=11.7 K 2024-11-15T14:14:46,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/bd1e0cbe249b44779d2d1eb02fd8f9fd as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/bd1e0cbe249b44779d2d1eb02fd8f9fd 2024-11-15T14:14:46,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/bd1e0cbe249b44779d2d1eb02fd8f9fd, entries=150, sequenceid=48, filesize=11.7 K 2024-11-15T14:14:46,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for 785aa89faccd90f1bafb73964185be68 in 830ms, sequenceid=48, compaction requested=false 2024-11-15T14:14:46,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:46,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T14:14:46,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:46,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:46,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:46,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:46,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:46,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:46,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:46,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:46,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:46,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:46,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/9b83c565e5c144ac9b23cc87cccb6f80 is 50, key is test_row_0/A:col10/1731680085462/Put/seqid=0 2024-11-15T14:14:46,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741843_1019 (size=14341) 2024-11-15T14:14:46,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680146332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:46,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:46,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:46,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:46,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680146350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680146355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680146351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680146355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680146463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680146493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680146504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680146504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680146505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:46,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:46,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:46,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:46,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:46,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:46,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:46,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:46,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/9b83c565e5c144ac9b23cc87cccb6f80 2024-11-15T14:14:46,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680146714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680146713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680146732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680146732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680146733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:46,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/b55713844aa545289aad03a16a4f1458 is 50, key is test_row_0/B:col10/1731680085462/Put/seqid=0 2024-11-15T14:14:46,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741844_1020 (size=12001) 2024-11-15T14:14:46,849 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T14:14:46,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:46,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:46,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:46,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:46,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,009 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T14:14:47,011 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T14:14:47,016 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T14:14:47,018 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T14:14:47,018 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T14:14:47,019 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T14:14:47,019 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T14:14:47,020 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T14:14:47,020 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T14:14:47,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:47,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:47,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:47,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:47,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680147042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680147043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:47,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:47,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680147052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:47,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680147062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680147063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:47,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:47,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:47,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:47,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:47,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/b55713844aa545289aad03a16a4f1458 2024-11-15T14:14:47,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d3c3ba32615041d8be12d6ef1424091e is 50, key is test_row_0/C:col10/1731680085462/Put/seqid=0 2024-11-15T14:14:47,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741845_1021 (size=12001) 2024-11-15T14:14:47,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:47,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:47,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:47,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:47,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:47,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:47,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:47,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:47,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680147563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:47,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680147563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:47,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680147579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:47,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680147582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:47,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680147583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:47,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:47,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:47,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:47,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d3c3ba32615041d8be12d6ef1424091e 2024-11-15T14:14:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T14:14:47,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/9b83c565e5c144ac9b23cc87cccb6f80 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/9b83c565e5c144ac9b23cc87cccb6f80 2024-11-15T14:14:47,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/9b83c565e5c144ac9b23cc87cccb6f80, entries=200, sequenceid=60, filesize=14.0 K 2024-11-15T14:14:47,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/b55713844aa545289aad03a16a4f1458 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b55713844aa545289aad03a16a4f1458 2024-11-15T14:14:47,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b55713844aa545289aad03a16a4f1458, entries=150, sequenceid=60, filesize=11.7 K 2024-11-15T14:14:47,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d3c3ba32615041d8be12d6ef1424091e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d3c3ba32615041d8be12d6ef1424091e 2024-11-15T14:14:47,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d3c3ba32615041d8be12d6ef1424091e, entries=150, sequenceid=60, filesize=11.7 K 2024-11-15T14:14:47,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 785aa89faccd90f1bafb73964185be68 in 1556ms, sequenceid=60, compaction requested=true 2024-11-15T14:14:47,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:47,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:14:47,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:47,767 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:47,767 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:47,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:14:47,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:47,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:14:47,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:47,771 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:47,772 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/B is initiating minor compaction (all files) 2024-11-15T14:14:47,773 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:47,773 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/B in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:47,773 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/A is initiating minor compaction (all files) 2024-11-15T14:14:47,773 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/A in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:47,773 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d51c9ea99b5c49f9ad521055e7433ff2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0be0ffb4d10e4ccaadce5c48cd8a951b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b55713844aa545289aad03a16a4f1458] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=35.2 K 2024-11-15T14:14:47,773 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/3388db958ff6440c8f0809727efcad5c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f5e8a24474f14395944f2895ed7d22f8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/9b83c565e5c144ac9b23cc87cccb6f80] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=42.0 K 2024-11-15T14:14:47,776 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting d51c9ea99b5c49f9ad521055e7433ff2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731680083627 2024-11-15T14:14:47,777 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0be0ffb4d10e4ccaadce5c48cd8a951b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731680084025 2024-11-15T14:14:47,779 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting b55713844aa545289aad03a16a4f1458, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731680085457 2024-11-15T14:14:47,781 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3388db958ff6440c8f0809727efcad5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731680083627 2024-11-15T14:14:47,785 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5e8a24474f14395944f2895ed7d22f8, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731680084025 2024-11-15T14:14:47,791 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b83c565e5c144ac9b23cc87cccb6f80, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731680085431 2024-11-15T14:14:47,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T14:14:47,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:47,818 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T14:14:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:47,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:47,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:47,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/2de211160af14e23bf93a2e510c00c12 is 50, key is test_row_0/A:col10/1731680086349/Put/seqid=0 2024-11-15T14:14:47,873 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#B#compaction#9 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:47,874 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#A#compaction#11 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:47,875 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/720d86a998f04e6d948dfce96409cde0 is 50, key is test_row_0/A:col10/1731680085462/Put/seqid=0 2024-11-15T14:14:47,880 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/0a5a7b75975e444c9dbc8106de536b0b is 50, key is test_row_0/B:col10/1731680085462/Put/seqid=0 2024-11-15T14:14:47,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741846_1022 (size=12001) 2024-11-15T14:14:47,932 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/2de211160af14e23bf93a2e510c00c12 2024-11-15T14:14:47,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741847_1023 (size=12104) 2024-11-15T14:14:48,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/afc71d668014404f90ceacc0c13e8433 is 50, key is test_row_0/B:col10/1731680086349/Put/seqid=0 2024-11-15T14:14:48,013 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/720d86a998f04e6d948dfce96409cde0 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/720d86a998f04e6d948dfce96409cde0 2024-11-15T14:14:48,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741848_1024 (size=12104) 2024-11-15T14:14:48,039 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/0a5a7b75975e444c9dbc8106de536b0b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0a5a7b75975e444c9dbc8106de536b0b 2024-11-15T14:14:48,064 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/B of 785aa89faccd90f1bafb73964185be68 into 0a5a7b75975e444c9dbc8106de536b0b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:48,064 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:48,065 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/A of 785aa89faccd90f1bafb73964185be68 into 720d86a998f04e6d948dfce96409cde0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:48,065 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:48,067 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/A, priority=13, startTime=1731680087763; duration=0sec 2024-11-15T14:14:48,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741849_1025 (size=12001) 2024-11-15T14:14:48,069 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/B, priority=13, startTime=1731680087767; duration=0sec 2024-11-15T14:14:48,070 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:48,070 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:B 2024-11-15T14:14:48,071 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:48,071 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/afc71d668014404f90ceacc0c13e8433 2024-11-15T14:14:48,079 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:48,079 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:A 2024-11-15T14:14:48,082 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:48,083 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/C is initiating minor compaction (all files) 2024-11-15T14:14:48,083 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/C in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:48,084 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/dc92041cfbfe4d058759c73642cafa5e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/bd1e0cbe249b44779d2d1eb02fd8f9fd, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d3c3ba32615041d8be12d6ef1424091e] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=35.2 K 2024-11-15T14:14:48,086 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting dc92041cfbfe4d058759c73642cafa5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731680083627 2024-11-15T14:14:48,090 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting bd1e0cbe249b44779d2d1eb02fd8f9fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731680084025 2024-11-15T14:14:48,092 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting d3c3ba32615041d8be12d6ef1424091e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731680085457 2024-11-15T14:14:48,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d1107b37e6d3442eac1f8370ade62825 is 50, key is test_row_0/C:col10/1731680086349/Put/seqid=0 2024-11-15T14:14:48,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741850_1026 (size=12001) 2024-11-15T14:14:48,150 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#C#compaction#14 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:48,151 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/2403f23cfd8c42b386dc853e5cd33c2e is 50, key is test_row_0/C:col10/1731680085462/Put/seqid=0 2024-11-15T14:14:48,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741851_1027 (size=12104) 2024-11-15T14:14:48,216 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/2403f23cfd8c42b386dc853e5cd33c2e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/2403f23cfd8c42b386dc853e5cd33c2e 2024-11-15T14:14:48,238 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/C of 785aa89faccd90f1bafb73964185be68 into 2403f23cfd8c42b386dc853e5cd33c2e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:48,238 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:48,238 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/C, priority=13, startTime=1731680087768; duration=0sec 2024-11-15T14:14:48,238 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:48,238 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:C 2024-11-15T14:14:48,546 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d1107b37e6d3442eac1f8370ade62825 2024-11-15T14:14:48,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/2de211160af14e23bf93a2e510c00c12 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/2de211160af14e23bf93a2e510c00c12 2024-11-15T14:14:48,574 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/2de211160af14e23bf93a2e510c00c12, entries=150, sequenceid=80, filesize=11.7 K 2024-11-15T14:14:48,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/afc71d668014404f90ceacc0c13e8433 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/afc71d668014404f90ceacc0c13e8433 2024-11-15T14:14:48,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:48,597 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/afc71d668014404f90ceacc0c13e8433, entries=150, sequenceid=80, filesize=11.7 K 2024-11-15T14:14:48,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d1107b37e6d3442eac1f8370ade62825 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d1107b37e6d3442eac1f8370ade62825 2024-11-15T14:14:48,626 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d1107b37e6d3442eac1f8370ade62825, entries=150, sequenceid=80, filesize=11.7 K 2024-11-15T14:14:48,629 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=53.67 KB/54960 for 785aa89faccd90f1bafb73964185be68 in 811ms, sequenceid=80, compaction requested=false 2024-11-15T14:14:48,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:48,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:48,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-15T14:14:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-15T14:14:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:48,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T14:14:48,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:48,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:48,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:48,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:48,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:48,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:48,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T14:14:48,640 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.0800 sec 2024-11-15T14:14:48,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 5.1100 sec 2024-11-15T14:14:48,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/264a6940dd7a4b9aaca2df2b7c0f88e7 is 50, key is test_row_0/A:col10/1731680088632/Put/seqid=0 2024-11-15T14:14:48,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741852_1028 (size=23705) 2024-11-15T14:14:48,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/264a6940dd7a4b9aaca2df2b7c0f88e7 2024-11-15T14:14:48,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/aa330313eb3c4b38bda31834edecae55 is 50, key is test_row_0/B:col10/1731680088632/Put/seqid=0 2024-11-15T14:14:48,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741853_1029 (size=12001) 2024-11-15T14:14:48,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/aa330313eb3c4b38bda31834edecae55 2024-11-15T14:14:48,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680148728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680148732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680148735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:48,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680148740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:48,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680148755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:48,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d9325afb22fc4893925cc7e91f709f6b is 50, key is test_row_0/C:col10/1731680088632/Put/seqid=0 2024-11-15T14:14:48,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741854_1030 (size=12001) 2024-11-15T14:14:48,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d9325afb22fc4893925cc7e91f709f6b 2024-11-15T14:14:48,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/264a6940dd7a4b9aaca2df2b7c0f88e7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/264a6940dd7a4b9aaca2df2b7c0f88e7 2024-11-15T14:14:48,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680148862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:48,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:48,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680148863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:48,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:48,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680148863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680148863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:48,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/264a6940dd7a4b9aaca2df2b7c0f88e7, entries=400, sequenceid=96, filesize=23.1 K 2024-11-15T14:14:48,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/aa330313eb3c4b38bda31834edecae55 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/aa330313eb3c4b38bda31834edecae55 2024-11-15T14:14:48,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/aa330313eb3c4b38bda31834edecae55, entries=150, sequenceid=96, filesize=11.7 K 2024-11-15T14:14:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680148892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d9325afb22fc4893925cc7e91f709f6b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d9325afb22fc4893925cc7e91f709f6b 2024-11-15T14:14:48,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d9325afb22fc4893925cc7e91f709f6b, entries=150, sequenceid=96, filesize=11.7 K 2024-11-15T14:14:48,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 785aa89faccd90f1bafb73964185be68 in 299ms, sequenceid=96, compaction requested=true 2024-11-15T14:14:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:48,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:14:48,932 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:48,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:48,932 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:48,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:14:48,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:48,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:14:48,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:48,936 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47810 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:48,936 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/A is initiating minor compaction (all files) 2024-11-15T14:14:48,936 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/A in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:48,936 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/720d86a998f04e6d948dfce96409cde0, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/2de211160af14e23bf93a2e510c00c12, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/264a6940dd7a4b9aaca2df2b7c0f88e7] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=46.7 K 2024-11-15T14:14:48,937 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:48,937 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/B is initiating minor compaction (all files) 2024-11-15T14:14:48,937 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/B in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:48,937 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0a5a7b75975e444c9dbc8106de536b0b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/afc71d668014404f90ceacc0c13e8433, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/aa330313eb3c4b38bda31834edecae55] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=35.3 K 2024-11-15T14:14:48,938 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 720d86a998f04e6d948dfce96409cde0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731680085457 2024-11-15T14:14:48,938 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a5a7b75975e444c9dbc8106de536b0b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731680085457 2024-11-15T14:14:48,939 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2de211160af14e23bf93a2e510c00c12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731680086338 2024-11-15T14:14:48,940 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 264a6940dd7a4b9aaca2df2b7c0f88e7, keycount=400, bloomtype=ROW, size=23.1 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731680088587 2024-11-15T14:14:48,940 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting afc71d668014404f90ceacc0c13e8433, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731680086338 2024-11-15T14:14:48,940 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting aa330313eb3c4b38bda31834edecae55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731680088619 2024-11-15T14:14:48,979 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#A#compaction#18 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:48,980 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/2e65ac0ab86e45d8a1c2527979418555 is 50, key is test_row_0/A:col10/1731680088632/Put/seqid=0 2024-11-15T14:14:48,988 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:48,989 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/4a098406aedc402cb726429e82ce79a7 is 50, key is test_row_0/B:col10/1731680088632/Put/seqid=0 2024-11-15T14:14:49,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741856_1032 (size=12207) 2024-11-15T14:14:49,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741855_1031 (size=12207) 2024-11-15T14:14:49,065 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/2e65ac0ab86e45d8a1c2527979418555 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/2e65ac0ab86e45d8a1c2527979418555 2024-11-15T14:14:49,085 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/A of 785aa89faccd90f1bafb73964185be68 into 2e65ac0ab86e45d8a1c2527979418555(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:49,086 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:49,086 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/A, priority=13, startTime=1731680088931; duration=0sec 2024-11-15T14:14:49,086 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:49,086 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:A 2024-11-15T14:14:49,086 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:49,090 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:49,091 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/C is initiating minor compaction (all files) 2024-11-15T14:14:49,091 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/C in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:49,091 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/2403f23cfd8c42b386dc853e5cd33c2e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d1107b37e6d3442eac1f8370ade62825, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d9325afb22fc4893925cc7e91f709f6b] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=35.3 K 2024-11-15T14:14:49,093 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2403f23cfd8c42b386dc853e5cd33c2e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731680085457 2024-11-15T14:14:49,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T14:14:49,094 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1107b37e6d3442eac1f8370ade62825, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731680086338 2024-11-15T14:14:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:49,096 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9325afb22fc4893925cc7e91f709f6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731680088619 2024-11-15T14:14:49,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680088730, max timestamp=1731680089094Num uniques 859; 2024-11-15T14:14:49,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:14:49,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680088730, max timestamp=1731680089094Num uniques 859; 2024-11-15T14:14:49,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:14:49,130 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#C#compaction#20 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:49,130 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/848a5d91bd504086b920192b0744202b is 50, key is test_row_0/C:col10/1731680088632/Put/seqid=0 2024-11-15T14:14:49,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:14:49,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680088730, max timestamp=1731680089094Num uniques 859; 2024-11-15T14:14:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:14:49,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:14:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:14:49,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/8f8e47d17b78473bb215ad031aba3b02 is 50, key is test_row_0/A:col10/1731680088741/Put/seqid=0 2024-11-15T14:14:49,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741857_1033 (size=12207) 2024-11-15T14:14:49,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741858_1034 (size=14341) 2024-11-15T14:14:49,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/8f8e47d17b78473bb215ad031aba3b02 2024-11-15T14:14:49,228 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/848a5d91bd504086b920192b0744202b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/848a5d91bd504086b920192b0744202b 2024-11-15T14:14:49,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/be2503f0d2514652ae1b4926a8a5b219 is 50, key is test_row_0/B:col10/1731680088741/Put/seqid=0 2024-11-15T14:14:49,259 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/C of 785aa89faccd90f1bafb73964185be68 into 848a5d91bd504086b920192b0744202b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:49,259 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:49,259 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/C, priority=13, startTime=1731680088935; duration=0sec 2024-11-15T14:14:49,261 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:49,261 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:C 2024-11-15T14:14:49,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741859_1035 (size=12001) 2024-11-15T14:14:49,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/be2503f0d2514652ae1b4926a8a5b219 2024-11-15T14:14:49,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680089189, max timestamp=1731680089281Num uniques 859; 2024-11-15T14:14:49,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:14:49,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731680089189, max timestamp=1731680089283Num uniques 859; 2024-11-15T14:14:49,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T14:14:49,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731680089189, max timestamp=1731680089290Num uniques 859; 2024-11-15T14:14:49,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T14:14:49,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d512c078f843489a940e7cb5f725854d is 50, key is test_row_0/C:col10/1731680088741/Put/seqid=0 2024-11-15T14:14:49,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741860_1036 (size=12001) 2024-11-15T14:14:49,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d512c078f843489a940e7cb5f725854d 2024-11-15T14:14:49,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731680089350, max timestamp=1731680089410Num uniques 1009; 2024-11-15T14:14:49,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731680089350, max timestamp=1731680089410Num uniques 1009; 2024-11-15T14:14:49,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=12268, suffix heap size=132904,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=268 2024-11-15T14:14:49,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=12268, suffix heap size=132904,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=268 2024-11-15T14:14:49,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/8f8e47d17b78473bb215ad031aba3b02 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/8f8e47d17b78473bb215ad031aba3b02 2024-11-15T14:14:49,433 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/4a098406aedc402cb726429e82ce79a7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/4a098406aedc402cb726429e82ce79a7 2024-11-15T14:14:49,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731680089350, max timestamp=1731680089407Num uniques 1018; 2024-11-15T14:14:49,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-15T14:14:49,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/8f8e47d17b78473bb215ad031aba3b02, entries=200, sequenceid=118, filesize=14.0 K 2024-11-15T14:14:49,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/be2503f0d2514652ae1b4926a8a5b219 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/be2503f0d2514652ae1b4926a8a5b219 2024-11-15T14:14:49,448 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/B of 785aa89faccd90f1bafb73964185be68 into 4a098406aedc402cb726429e82ce79a7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:49,448 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:49,448 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/B, priority=13, startTime=1731680088932; duration=0sec 2024-11-15T14:14:49,448 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:49,448 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:B 2024-11-15T14:14:49,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/be2503f0d2514652ae1b4926a8a5b219, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T14:14:49,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d512c078f843489a940e7cb5f725854d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d512c078f843489a940e7cb5f725854d 2024-11-15T14:14:49,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d512c078f843489a940e7cb5f725854d, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T14:14:49,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=100.63 KB/103050 for 785aa89faccd90f1bafb73964185be68 in 387ms, sequenceid=118, compaction requested=false 2024-11-15T14:14:49,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:49,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=107.34 KB heapSize=295.08 KB 2024-11-15T14:14:49,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:14:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:14:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:14:49,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/dddc2ad8482c48658bb1b76cfc424e6b is 50, key is test_row_0/A:col10/1731680089445/Put/seqid=0 2024-11-15T14:14:49,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741861_1037 (size=21715) 2024-11-15T14:14:49,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/dddc2ad8482c48658bb1b76cfc424e6b 2024-11-15T14:14:49,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680149575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:49,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:49,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680149579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:49,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/5194dc1a29c94dc592f124c2383e9786 is 50, key is test_row_0/B:col10/1731680089445/Put/seqid=0 2024-11-15T14:14:49,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680149581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680149583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:49,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:49,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680149595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:49,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741862_1038 (size=12151) 2024-11-15T14:14:49,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/5194dc1a29c94dc592f124c2383e9786 2024-11-15T14:14:49,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/572064a882744c878d64acaa1ba41492 is 50, key is test_row_0/C:col10/1731680089445/Put/seqid=0 2024-11-15T14:14:49,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680149702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:49,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741863_1039 (size=12151) 2024-11-15T14:14:49,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680149703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:49,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/572064a882744c878d64acaa1ba41492 2024-11-15T14:14:49,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/dddc2ad8482c48658bb1b76cfc424e6b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/dddc2ad8482c48658bb1b76cfc424e6b 2024-11-15T14:14:49,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680149712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:49,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:49,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680149712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:49,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/dddc2ad8482c48658bb1b76cfc424e6b, entries=350, sequenceid=169, filesize=21.2 K 2024-11-15T14:14:49,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/5194dc1a29c94dc592f124c2383e9786 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/5194dc1a29c94dc592f124c2383e9786 2024-11-15T14:14:49,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680149722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:49,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/5194dc1a29c94dc592f124c2383e9786, entries=150, sequenceid=169, filesize=11.9 K 2024-11-15T14:14:49,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/572064a882744c878d64acaa1ba41492 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/572064a882744c878d64acaa1ba41492 2024-11-15T14:14:49,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/572064a882744c878d64acaa1ba41492, entries=150, sequenceid=169, filesize=11.9 K 2024-11-15T14:14:49,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~314.95 KB/322512, currentSize=67.09 KB/68700 for 785aa89faccd90f1bafb73964185be68 in 306ms, sequenceid=169, compaction requested=true 2024-11-15T14:14:49,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:49,792 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:49,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:14:49,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:49,793 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:49,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:14:49,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:49,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:14:49,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:49,795 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 48263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:49,795 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/A is initiating minor compaction (all files) 2024-11-15T14:14:49,795 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/A in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:49,795 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/2e65ac0ab86e45d8a1c2527979418555, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/8f8e47d17b78473bb215ad031aba3b02, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/dddc2ad8482c48658bb1b76cfc424e6b] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=47.1 K 2024-11-15T14:14:49,796 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:49,796 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/B is initiating minor compaction (all files) 2024-11-15T14:14:49,797 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/B in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:49,797 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/4a098406aedc402cb726429e82ce79a7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/be2503f0d2514652ae1b4926a8a5b219, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/5194dc1a29c94dc592f124c2383e9786] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=35.5 K 2024-11-15T14:14:49,797 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e65ac0ab86e45d8a1c2527979418555, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731680088619 2024-11-15T14:14:49,798 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a098406aedc402cb726429e82ce79a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731680088619 2024-11-15T14:14:49,798 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f8e47d17b78473bb215ad031aba3b02, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731680088730 2024-11-15T14:14:49,800 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting be2503f0d2514652ae1b4926a8a5b219, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731680088741 2024-11-15T14:14:49,800 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5194dc1a29c94dc592f124c2383e9786, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731680089445 2024-11-15T14:14:49,801 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting dddc2ad8482c48658bb1b76cfc424e6b, keycount=350, bloomtype=ROW, size=21.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731680089445 2024-11-15T14:14:49,834 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:49,839 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/93d4d14ab5ce42e0b712fed8d7adb995 is 50, key is test_row_0/B:col10/1731680089445/Put/seqid=0 2024-11-15T14:14:49,847 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#A#compaction#28 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:49,848 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/cb34c96bca1e4d57ac3f13a9fe858695 is 50, key is test_row_0/A:col10/1731680089445/Put/seqid=0 2024-11-15T14:14:49,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741864_1040 (size=12459) 2024-11-15T14:14:49,908 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/93d4d14ab5ce42e0b712fed8d7adb995 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/93d4d14ab5ce42e0b712fed8d7adb995 2024-11-15T14:14:49,924 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/B of 785aa89faccd90f1bafb73964185be68 into 93d4d14ab5ce42e0b712fed8d7adb995(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:49,924 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:49,924 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/B, priority=13, startTime=1731680089793; duration=0sec 2024-11-15T14:14:49,925 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:49,925 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:B 2024-11-15T14:14:49,925 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:49,927 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:49,927 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/C is initiating minor compaction (all files) 2024-11-15T14:14:49,927 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/C in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:49,927 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/848a5d91bd504086b920192b0744202b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d512c078f843489a940e7cb5f725854d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/572064a882744c878d64acaa1ba41492] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=35.5 K 2024-11-15T14:14:49,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741865_1041 (size=12459) 2024-11-15T14:14:49,930 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 848a5d91bd504086b920192b0744202b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731680088619 2024-11-15T14:14:49,931 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting d512c078f843489a940e7cb5f725854d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731680088741 2024-11-15T14:14:49,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T14:14:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:49,934 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 572064a882744c878d64acaa1ba41492, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731680089445 2024-11-15T14:14:49,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:49,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:49,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:49,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:49,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:49,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:49,948 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/cb34c96bca1e4d57ac3f13a9fe858695 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/cb34c96bca1e4d57ac3f13a9fe858695 2024-11-15T14:14:49,954 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#C#compaction#29 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:49,955 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/eb7ca1834544454091b8fa84805bcca6 is 50, key is test_row_0/C:col10/1731680089445/Put/seqid=0 2024-11-15T14:14:50,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/6d0a08c1ddec4a99bf42203e6845b572 is 50, key is test_row_0/A:col10/1731680089583/Put/seqid=0 2024-11-15T14:14:50,025 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/A of 785aa89faccd90f1bafb73964185be68 into cb34c96bca1e4d57ac3f13a9fe858695(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:50,026 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:50,026 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/A, priority=13, startTime=1731680089792; duration=0sec 2024-11-15T14:14:50,026 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:50,026 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:A 2024-11-15T14:14:50,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741866_1042 (size=12459) 2024-11-15T14:14:50,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680150028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,062 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/eb7ca1834544454091b8fa84805bcca6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/eb7ca1834544454091b8fa84805bcca6 2024-11-15T14:14:50,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680150044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,078 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/C of 785aa89faccd90f1bafb73964185be68 into eb7ca1834544454091b8fa84805bcca6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:50,078 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:50,078 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/C, priority=13, startTime=1731680089793; duration=0sec 2024-11-15T14:14:50,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,079 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:50,079 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:C 2024-11-15T14:14:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680150044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680150049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680150050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741867_1043 (size=16931) 2024-11-15T14:14:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680150162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680150183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680150192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680150192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680150194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680150382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680150391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680150402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680150402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680150402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/6d0a08c1ddec4a99bf42203e6845b572 2024-11-15T14:14:50,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/75e66dd8de9145108929385c45f12a04 is 50, key is test_row_0/B:col10/1731680089583/Put/seqid=0 2024-11-15T14:14:50,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741868_1044 (size=12151) 2024-11-15T14:14:50,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680150702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680150712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680150712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680150713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680150713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:50,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/75e66dd8de9145108929385c45f12a04 2024-11-15T14:14:51,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/c6e8a6fae42c45b3bfbb4bf26cfe0b11 is 50, key is test_row_0/C:col10/1731680089583/Put/seqid=0 2024-11-15T14:14:51,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741869_1045 (size=12151) 2024-11-15T14:14:51,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680151222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:51,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680151233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:51,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680151233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:51,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680151233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:51,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680151233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:51,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/c6e8a6fae42c45b3bfbb4bf26cfe0b11 2024-11-15T14:14:51,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/6d0a08c1ddec4a99bf42203e6845b572 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/6d0a08c1ddec4a99bf42203e6845b572 2024-11-15T14:14:51,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/6d0a08c1ddec4a99bf42203e6845b572, entries=250, sequenceid=185, filesize=16.5 K 2024-11-15T14:14:51,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/75e66dd8de9145108929385c45f12a04 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/75e66dd8de9145108929385c45f12a04 2024-11-15T14:14:51,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/75e66dd8de9145108929385c45f12a04, entries=150, sequenceid=185, filesize=11.9 K 2024-11-15T14:14:51,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/c6e8a6fae42c45b3bfbb4bf26cfe0b11 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/c6e8a6fae42c45b3bfbb4bf26cfe0b11 2024-11-15T14:14:51,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/c6e8a6fae42c45b3bfbb4bf26cfe0b11, entries=150, sequenceid=185, filesize=11.9 K 2024-11-15T14:14:51,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 785aa89faccd90f1bafb73964185be68 in 1648ms, sequenceid=185, compaction requested=false 2024-11-15T14:14:51,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T14:14:51,743 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:14:51,744 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:14:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-15T14:14:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T14:14:51,749 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:14:51,751 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:14:51,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:14:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T14:14:51,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T14:14:51,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:51,906 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T14:14:51,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:51,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:51,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:51,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:51,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:51,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:51,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/24233f863197493ea465e70803ab3220 is 50, key is test_row_0/A:col10/1731680090023/Put/seqid=0 2024-11-15T14:14:51,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741870_1046 (size=12151) 2024-11-15T14:14:51,974 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/24233f863197493ea465e70803ab3220 2024-11-15T14:14:52,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/4a8db4d13b6b4b19826607ae853aad28 is 50, key is test_row_0/B:col10/1731680090023/Put/seqid=0 2024-11-15T14:14:52,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741871_1047 (size=12151) 2024-11-15T14:14:52,029 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/4a8db4d13b6b4b19826607ae853aad28 2024-11-15T14:14:52,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d460826e8ae044e9ac76c4e2f972161b is 50, key is test_row_0/C:col10/1731680090023/Put/seqid=0 2024-11-15T14:14:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T14:14:52,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741872_1048 (size=12151) 2024-11-15T14:14:52,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:52,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680152319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680152319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680152321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680152329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680152332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T14:14:52,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680152442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680152442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680152443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680152453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680152453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,481 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d460826e8ae044e9ac76c4e2f972161b 2024-11-15T14:14:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/24233f863197493ea465e70803ab3220 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/24233f863197493ea465e70803ab3220 2024-11-15T14:14:52,505 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/24233f863197493ea465e70803ab3220, entries=150, sequenceid=205, filesize=11.9 K 2024-11-15T14:14:52,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/4a8db4d13b6b4b19826607ae853aad28 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/4a8db4d13b6b4b19826607ae853aad28 2024-11-15T14:14:52,519 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/4a8db4d13b6b4b19826607ae853aad28, entries=150, sequenceid=205, filesize=11.9 K 2024-11-15T14:14:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d460826e8ae044e9ac76c4e2f972161b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d460826e8ae044e9ac76c4e2f972161b 2024-11-15T14:14:52,535 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d460826e8ae044e9ac76c4e2f972161b, entries=150, sequenceid=205, filesize=11.9 K 2024-11-15T14:14:52,536 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 785aa89faccd90f1bafb73964185be68 in 631ms, sequenceid=205, compaction requested=true 2024-11-15T14:14:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:52,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-15T14:14:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-15T14:14:52,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-15T14:14:52,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 788 msec 2024-11-15T14:14:52,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 803 msec 2024-11-15T14:14:52,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T14:14:52,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:52,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:52,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:52,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:52,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:52,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:52,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/703f1556c47542049469e8f4ea4f7564 is 50, key is test_row_0/A:col10/1731680092657/Put/seqid=0 2024-11-15T14:14:52,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680152714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680152714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680152721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741873_1049 (size=12151) 2024-11-15T14:14:52,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680152734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680152736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680152844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680152844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680152844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680152863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680152863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T14:14:52,882 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:14:52,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:14:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-15T14:14:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T14:14:52,886 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:14:52,888 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:14:52,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:14:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T14:14:53,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T14:14:53,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:53,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:53,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:53,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:53,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:53,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680153063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680153064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680153083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680153083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680153084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/703f1556c47542049469e8f4ea4f7564 2024-11-15T14:14:53,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/b3fadf8a5610405b994a08ea85da1b0f is 50, key is test_row_0/B:col10/1731680092657/Put/seqid=0 2024-11-15T14:14:53,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T14:14:53,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:53,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:53,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:53,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:53,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T14:14:53,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741874_1050 (size=12151) 2024-11-15T14:14:53,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/b3fadf8a5610405b994a08ea85da1b0f 2024-11-15T14:14:53,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/0dfbc9fa356848958cb12b4d9112a252 is 50, key is test_row_0/C:col10/1731680092657/Put/seqid=0 2024-11-15T14:14:53,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741875_1051 (size=12151) 2024-11-15T14:14:53,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/0dfbc9fa356848958cb12b4d9112a252 2024-11-15T14:14:53,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/703f1556c47542049469e8f4ea4f7564 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/703f1556c47542049469e8f4ea4f7564 2024-11-15T14:14:53,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/703f1556c47542049469e8f4ea4f7564, entries=150, sequenceid=220, filesize=11.9 K 2024-11-15T14:14:53,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/b3fadf8a5610405b994a08ea85da1b0f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b3fadf8a5610405b994a08ea85da1b0f 2024-11-15T14:14:53,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b3fadf8a5610405b994a08ea85da1b0f, entries=150, sequenceid=220, filesize=11.9 K 2024-11-15T14:14:53,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/0dfbc9fa356848958cb12b4d9112a252 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/0dfbc9fa356848958cb12b4d9112a252 2024-11-15T14:14:53,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/0dfbc9fa356848958cb12b4d9112a252, entries=150, sequenceid=220, filesize=11.9 K 2024-11-15T14:14:53,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 785aa89faccd90f1bafb73964185be68 in 687ms, sequenceid=220, compaction requested=true 2024-11-15T14:14:53,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:53,348 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:14:53,351 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53692 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:14:53,351 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/A is initiating minor compaction (all files) 2024-11-15T14:14:53,351 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/A in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:53,352 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/cb34c96bca1e4d57ac3f13a9fe858695, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/6d0a08c1ddec4a99bf42203e6845b572, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/24233f863197493ea465e70803ab3220, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/703f1556c47542049469e8f4ea4f7564] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=52.4 K 2024-11-15T14:14:53,353 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb34c96bca1e4d57ac3f13a9fe858695, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731680089445 2024-11-15T14:14:53,354 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d0a08c1ddec4a99bf42203e6845b572, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731680089511 2024-11-15T14:14:53,355 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24233f863197493ea465e70803ab3220, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731680090023 2024-11-15T14:14:53,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:14:53,355 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 703f1556c47542049469e8f4ea4f7564, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731680092312 2024-11-15T14:14:53,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:53,357 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:14:53,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T14:14:53,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:53,359 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T14:14:53,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:14:53,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:53,361 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:14:53,361 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/B is initiating minor compaction (all files) 2024-11-15T14:14:53,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:14:53,361 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/B in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:53,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:53,362 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/93d4d14ab5ce42e0b712fed8d7adb995, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/75e66dd8de9145108929385c45f12a04, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/4a8db4d13b6b4b19826607ae853aad28, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b3fadf8a5610405b994a08ea85da1b0f] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=47.8 K 2024-11-15T14:14:53,363 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 93d4d14ab5ce42e0b712fed8d7adb995, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731680089445 2024-11-15T14:14:53,366 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 75e66dd8de9145108929385c45f12a04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731680089580 2024-11-15T14:14:53,367 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a8db4d13b6b4b19826607ae853aad28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731680090023 2024-11-15T14:14:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:53,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:53,368 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting b3fadf8a5610405b994a08ea85da1b0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731680092312 2024-11-15T14:14:53,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/072658d744a9464fb547b64d76b19244 is 50, key is test_row_0/A:col10/1731680092718/Put/seqid=0 2024-11-15T14:14:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:53,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:53,400 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#A#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:53,401 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/f70ec0bf132e4f53ae1d677c81532847 is 50, key is test_row_0/A:col10/1731680092657/Put/seqid=0 2024-11-15T14:14:53,411 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#B#compaction#41 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:53,412 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/e6e5bec6e9da402794a38111eb9c0945 is 50, key is test_row_0/B:col10/1731680092657/Put/seqid=0 2024-11-15T14:14:53,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680153421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680153423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741876_1052 (size=12151) 2024-11-15T14:14:53,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680153435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680153437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680153439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,475 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/072658d744a9464fb547b64d76b19244 2024-11-15T14:14:53,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741877_1053 (size=12595) 2024-11-15T14:14:53,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741878_1054 (size=12595) 2024-11-15T14:14:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T14:14:53,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/1a29e60500ef4b95a4786531ad72e2c8 is 50, key is test_row_0/B:col10/1731680092718/Put/seqid=0 2024-11-15T14:14:53,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680153541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680153552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741879_1055 (size=12151) 2024-11-15T14:14:53,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680153562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680153562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680153563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680153752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680153763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680153782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680153782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680153782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:53,892 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/e6e5bec6e9da402794a38111eb9c0945 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/e6e5bec6e9da402794a38111eb9c0945 2024-11-15T14:14:53,892 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/f70ec0bf132e4f53ae1d677c81532847 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f70ec0bf132e4f53ae1d677c81532847 2024-11-15T14:14:53,903 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 785aa89faccd90f1bafb73964185be68/B of 785aa89faccd90f1bafb73964185be68 into e6e5bec6e9da402794a38111eb9c0945(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:53,903 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:53,903 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/B, priority=12, startTime=1731680093357; duration=0sec 2024-11-15T14:14:53,903 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:53,903 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:B 2024-11-15T14:14:53,903 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:14:53,906 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:14:53,906 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/C is initiating minor compaction (all files) 2024-11-15T14:14:53,907 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/C in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:53,907 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/eb7ca1834544454091b8fa84805bcca6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/c6e8a6fae42c45b3bfbb4bf26cfe0b11, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d460826e8ae044e9ac76c4e2f972161b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/0dfbc9fa356848958cb12b4d9112a252] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=47.8 K 2024-11-15T14:14:53,907 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting eb7ca1834544454091b8fa84805bcca6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731680089445 2024-11-15T14:14:53,908 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e8a6fae42c45b3bfbb4bf26cfe0b11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731680089580 2024-11-15T14:14:53,909 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting d460826e8ae044e9ac76c4e2f972161b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731680090023 2024-11-15T14:14:53,909 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dfbc9fa356848958cb12b4d9112a252, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731680092312 2024-11-15T14:14:53,910 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 785aa89faccd90f1bafb73964185be68/A of 785aa89faccd90f1bafb73964185be68 into f70ec0bf132e4f53ae1d677c81532847(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:53,910 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:53,910 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/A, priority=12, startTime=1731680093347; duration=0sec 2024-11-15T14:14:53,910 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:53,910 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:A 2024-11-15T14:14:53,931 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#C#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:53,932 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d9589d28c0f1491a92e3c0a2599329c1 is 50, key is test_row_0/C:col10/1731680092657/Put/seqid=0 2024-11-15T14:14:53,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741880_1056 (size=12595) 2024-11-15T14:14:53,959 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/1a29e60500ef4b95a4786531ad72e2c8 2024-11-15T14:14:53,963 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/d9589d28c0f1491a92e3c0a2599329c1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d9589d28c0f1491a92e3c0a2599329c1 2024-11-15T14:14:53,973 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 785aa89faccd90f1bafb73964185be68/C of 785aa89faccd90f1bafb73964185be68 into d9589d28c0f1491a92e3c0a2599329c1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:53,973 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:53,973 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/C, priority=12, startTime=1731680093360; duration=0sec 2024-11-15T14:14:53,973 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:53,973 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:C 2024-11-15T14:14:53,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/3e2c09ad12434c91879f4293e0f89b36 is 50, key is test_row_0/C:col10/1731680092718/Put/seqid=0 2024-11-15T14:14:54,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741881_1057 (size=12151) 2024-11-15T14:14:54,003 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/3e2c09ad12434c91879f4293e0f89b36 2024-11-15T14:14:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T14:14:54,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/072658d744a9464fb547b64d76b19244 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/072658d744a9464fb547b64d76b19244 2024-11-15T14:14:54,036 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/072658d744a9464fb547b64d76b19244, entries=150, sequenceid=238, filesize=11.9 K 2024-11-15T14:14:54,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/1a29e60500ef4b95a4786531ad72e2c8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/1a29e60500ef4b95a4786531ad72e2c8 2024-11-15T14:14:54,046 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/1a29e60500ef4b95a4786531ad72e2c8, entries=150, sequenceid=238, filesize=11.9 K 2024-11-15T14:14:54,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/3e2c09ad12434c91879f4293e0f89b36 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/3e2c09ad12434c91879f4293e0f89b36 2024-11-15T14:14:54,067 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/3e2c09ad12434c91879f4293e0f89b36, entries=150, sequenceid=238, filesize=11.9 K 2024-11-15T14:14:54,071 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 785aa89faccd90f1bafb73964185be68 in 711ms, sequenceid=238, compaction requested=false 2024-11-15T14:14:54,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:54,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:54,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-15T14:14:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-15T14:14:54,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-15T14:14:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:54,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1860 sec 2024-11-15T14:14:54,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T14:14:54,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:54,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:54,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:54,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:54,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:54,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:54,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/f38e14d47b1b4a53b72e07bfc4f141d8 is 50, key is test_row_0/A:col10/1731680093414/Put/seqid=0 2024-11-15T14:14:54,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.2100 sec 2024-11-15T14:14:54,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741882_1058 (size=14541) 2024-11-15T14:14:54,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680154128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/f38e14d47b1b4a53b72e07bfc4f141d8 2024-11-15T14:14:54,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680154131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680154135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680154135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680154142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/897caf6eeec34761885f0b83b538381d is 50, key is test_row_0/B:col10/1731680093414/Put/seqid=0 2024-11-15T14:14:54,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741883_1059 (size=12151) 2024-11-15T14:14:54,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680154253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680154253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680154254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680154253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680154262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680154473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680154473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680154473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680154473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680154473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/897caf6eeec34761885f0b83b538381d 2024-11-15T14:14:54,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/9274949692c8424894fc59024a4f01b3 is 50, key is test_row_0/C:col10/1731680093414/Put/seqid=0 2024-11-15T14:14:54,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741884_1060 (size=12151) 2024-11-15T14:14:54,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/9274949692c8424894fc59024a4f01b3 2024-11-15T14:14:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/f38e14d47b1b4a53b72e07bfc4f141d8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f38e14d47b1b4a53b72e07bfc4f141d8 2024-11-15T14:14:54,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f38e14d47b1b4a53b72e07bfc4f141d8, entries=200, sequenceid=256, filesize=14.2 K 2024-11-15T14:14:54,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/897caf6eeec34761885f0b83b538381d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/897caf6eeec34761885f0b83b538381d 2024-11-15T14:14:54,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/897caf6eeec34761885f0b83b538381d, entries=150, sequenceid=256, filesize=11.9 K 2024-11-15T14:14:54,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/9274949692c8424894fc59024a4f01b3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/9274949692c8424894fc59024a4f01b3 2024-11-15T14:14:54,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/9274949692c8424894fc59024a4f01b3, entries=150, sequenceid=256, filesize=11.9 K 2024-11-15T14:14:54,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 785aa89faccd90f1bafb73964185be68 in 634ms, sequenceid=256, compaction requested=true 2024-11-15T14:14:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:14:54,722 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:54,723 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:54,724 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:14:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:54,724 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/A is initiating minor compaction (all files) 2024-11-15T14:14:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:14:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:54,724 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/A in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:54,724 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f70ec0bf132e4f53ae1d677c81532847, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/072658d744a9464fb547b64d76b19244, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f38e14d47b1b4a53b72e07bfc4f141d8] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=38.4 K 2024-11-15T14:14:54,724 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f70ec0bf132e4f53ae1d677c81532847, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731680092312 2024-11-15T14:14:54,725 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:54,725 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/B is initiating minor compaction (all files) 2024-11-15T14:14:54,725 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/B in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:54,725 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/e6e5bec6e9da402794a38111eb9c0945, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/1a29e60500ef4b95a4786531ad72e2c8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/897caf6eeec34761885f0b83b538381d] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=36.0 K 2024-11-15T14:14:54,726 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 072658d744a9464fb547b64d76b19244, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731680092708 2024-11-15T14:14:54,726 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting e6e5bec6e9da402794a38111eb9c0945, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731680092312 2024-11-15T14:14:54,726 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f38e14d47b1b4a53b72e07bfc4f141d8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731680093414 2024-11-15T14:14:54,726 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a29e60500ef4b95a4786531ad72e2c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731680092708 2024-11-15T14:14:54,727 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 897caf6eeec34761885f0b83b538381d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731680093414 2024-11-15T14:14:54,761 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:54,762 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:54,762 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/c281b18e4c2b406f82133a3a092b6506 is 50, key is test_row_0/B:col10/1731680093414/Put/seqid=0 2024-11-15T14:14:54,762 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/f3edf02cd9cf4f7e8aaec7a6e3626075 is 50, key is test_row_0/A:col10/1731680093414/Put/seqid=0 2024-11-15T14:14:54,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T14:14:54,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:54,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:54,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:54,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:54,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:54,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:54,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741885_1061 (size=12697) 2024-11-15T14:14:54,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741886_1062 (size=12697) 2024-11-15T14:14:54,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680154829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/ece81dd4bc5f4483a6677388e681a842 is 50, key is test_row_0/A:col10/1731680094797/Put/seqid=0 2024-11-15T14:14:54,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680154837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680154838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680154839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680154839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741887_1063 (size=12301) 2024-11-15T14:14:54,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/ece81dd4bc5f4483a6677388e681a842 2024-11-15T14:14:54,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/7a79691256084b12b65f23b5fcd070a1 is 50, key is test_row_0/B:col10/1731680094797/Put/seqid=0 2024-11-15T14:14:54,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741888_1064 (size=12301) 2024-11-15T14:14:54,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/7a79691256084b12b65f23b5fcd070a1 2024-11-15T14:14:54,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680154952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680154952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680154962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680154963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680154963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:54,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:54,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/e7bafdccee2240cd893151bd139d0301 is 50, key is test_row_0/C:col10/1731680094797/Put/seqid=0 2024-11-15T14:14:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T14:14:55,038 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:14:55,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:14:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-15T14:14:55,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741889_1065 (size=12301) 2024-11-15T14:14:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T14:14:55,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/e7bafdccee2240cd893151bd139d0301 2024-11-15T14:14:55,041 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:14:55,043 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:14:55,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:14:55,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/ece81dd4bc5f4483a6677388e681a842 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/ece81dd4bc5f4483a6677388e681a842 2024-11-15T14:14:55,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/ece81dd4bc5f4483a6677388e681a842, entries=150, sequenceid=274, filesize=12.0 K 2024-11-15T14:14:55,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/7a79691256084b12b65f23b5fcd070a1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/7a79691256084b12b65f23b5fcd070a1 2024-11-15T14:14:55,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/7a79691256084b12b65f23b5fcd070a1, entries=150, sequenceid=274, filesize=12.0 K 2024-11-15T14:14:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/e7bafdccee2240cd893151bd139d0301 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/e7bafdccee2240cd893151bd139d0301 2024-11-15T14:14:55,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/e7bafdccee2240cd893151bd139d0301, entries=150, sequenceid=274, filesize=12.0 K 2024-11-15T14:14:55,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 785aa89faccd90f1bafb73964185be68 in 294ms, sequenceid=274, compaction requested=true 2024-11-15T14:14:55,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:55,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:A, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:14:55,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:14:55,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:B, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:14:55,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T14:14:55,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:14:55,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T14:14:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T14:14:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:55,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T14:14:55,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:55,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:55,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:55,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:55,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:55,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:55,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T14:14:55,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:55,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:55,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:55,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:55,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:55,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/1360819225d54047b263455d8324460d is 50, key is test_row_0/A:col10/1731680095177/Put/seqid=0 2024-11-15T14:14:55,228 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/c281b18e4c2b406f82133a3a092b6506 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/c281b18e4c2b406f82133a3a092b6506 2024-11-15T14:14:55,238 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/B of 785aa89faccd90f1bafb73964185be68 into c281b18e4c2b406f82133a3a092b6506(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:55,238 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:55,238 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/B, priority=13, startTime=1731680094722; duration=0sec 2024-11-15T14:14:55,239 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T14:14:55,239 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:B 2024-11-15T14:14:55,239 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-15T14:14:55,240 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T14:14:55,241 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T14:14:55,241 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. because compaction request was cancelled 2024-11-15T14:14:55,241 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:A 2024-11-15T14:14:55,242 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:14:55,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741890_1066 (size=12301) 2024-11-15T14:14:55,248 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:14:55,248 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/C is initiating minor compaction (all files) 2024-11-15T14:14:55,248 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/C in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:55,249 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d9589d28c0f1491a92e3c0a2599329c1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/3e2c09ad12434c91879f4293e0f89b36, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/9274949692c8424894fc59024a4f01b3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/e7bafdccee2240cd893151bd139d0301] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=48.0 K 2024-11-15T14:14:55,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/1360819225d54047b263455d8324460d 2024-11-15T14:14:55,257 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting d9589d28c0f1491a92e3c0a2599329c1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731680092312 2024-11-15T14:14:55,259 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e2c09ad12434c91879f4293e0f89b36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731680092708 2024-11-15T14:14:55,260 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/f3edf02cd9cf4f7e8aaec7a6e3626075 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f3edf02cd9cf4f7e8aaec7a6e3626075 2024-11-15T14:14:55,261 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9274949692c8424894fc59024a4f01b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731680093414 2024-11-15T14:14:55,263 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting e7bafdccee2240cd893151bd139d0301, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731680094123 2024-11-15T14:14:55,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/0d0249d90a9545cc8311fa0baa17408e is 50, key is test_row_0/B:col10/1731680095177/Put/seqid=0 2024-11-15T14:14:55,280 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#C#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:55,281 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/8b7afb2cce8c4bf181bc8b1c81f85904 is 50, key is test_row_0/C:col10/1731680094797/Put/seqid=0 2024-11-15T14:14:55,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680155240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680155245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,291 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/A of 785aa89faccd90f1bafb73964185be68 into f3edf02cd9cf4f7e8aaec7a6e3626075(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:55,291 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:55,291 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/A, priority=13, startTime=1731680094722; duration=0sec 2024-11-15T14:14:55,291 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:14:55,291 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:A 2024-11-15T14:14:55,292 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:B 2024-11-15T14:14:55,292 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-15T14:14:55,292 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T14:14:55,292 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T14:14:55,292 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. because compaction request was cancelled 2024-11-15T14:14:55,292 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:C 2024-11-15T14:14:55,292 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T14:14:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,295 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T14:14:55,295 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T14:14:55,295 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. because compaction request was cancelled 2024-11-15T14:14:55,295 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:B 2024-11-15T14:14:55,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680155285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680155291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680155295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741891_1067 (size=12881) 2024-11-15T14:14:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T14:14:55,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741892_1068 (size=12301) 2024-11-15T14:14:55,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T14:14:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:55,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:55,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:55,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/0d0249d90a9545cc8311fa0baa17408e 2024-11-15T14:14:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:55,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680155392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/fac5980c864440759781fb4675c33763 is 50, key is test_row_0/C:col10/1731680095177/Put/seqid=0 2024-11-15T14:14:55,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680155402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680155412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680155414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680155414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741893_1069 (size=12301) 2024-11-15T14:14:55,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/fac5980c864440759781fb4675c33763 2024-11-15T14:14:55,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/1360819225d54047b263455d8324460d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/1360819225d54047b263455d8324460d 2024-11-15T14:14:55,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/1360819225d54047b263455d8324460d, entries=150, sequenceid=291, filesize=12.0 K 2024-11-15T14:14:55,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/0d0249d90a9545cc8311fa0baa17408e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0d0249d90a9545cc8311fa0baa17408e 2024-11-15T14:14:55,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0d0249d90a9545cc8311fa0baa17408e, entries=150, sequenceid=291, filesize=12.0 K 2024-11-15T14:14:55,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/fac5980c864440759781fb4675c33763 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fac5980c864440759781fb4675c33763 2024-11-15T14:14:55,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fac5980c864440759781fb4675c33763, entries=150, sequenceid=291, filesize=12.0 K 2024-11-15T14:14:55,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 785aa89faccd90f1bafb73964185be68 in 326ms, sequenceid=291, compaction requested=true 2024-11-15T14:14:55,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:55,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:14:55,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:55,508 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:55,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:14:55,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:55,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:14:55,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:14:55,510 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:55,510 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/A is initiating minor compaction (all files) 2024-11-15T14:14:55,510 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/A in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:55,510 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f3edf02cd9cf4f7e8aaec7a6e3626075, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/ece81dd4bc5f4483a6677388e681a842, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/1360819225d54047b263455d8324460d] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=36.4 K 2024-11-15T14:14:55,511 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3edf02cd9cf4f7e8aaec7a6e3626075, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731680093414 2024-11-15T14:14:55,512 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ece81dd4bc5f4483a6677388e681a842, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731680094123 2024-11-15T14:14:55,512 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1360819225d54047b263455d8324460d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731680094817 2024-11-15T14:14:55,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T14:14:55,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:55,529 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T14:14:55,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:55,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:55,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:55,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:55,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:55,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:55,534 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#A#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:55,535 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/f9a8f0fe6ffd42f9884bb182035c4ec4 is 50, key is test_row_0/A:col10/1731680095177/Put/seqid=0 2024-11-15T14:14:55,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/ad5fe519b1d74e70b8df70ac8982460d is 50, key is test_row_0/A:col10/1731680095248/Put/seqid=0 2024-11-15T14:14:55,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741894_1070 (size=12949) 2024-11-15T14:14:55,614 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/f9a8f0fe6ffd42f9884bb182035c4ec4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f9a8f0fe6ffd42f9884bb182035c4ec4 2024-11-15T14:14:55,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:55,625 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/A of 785aa89faccd90f1bafb73964185be68 into f9a8f0fe6ffd42f9884bb182035c4ec4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:55,625 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:55,625 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/A, priority=13, startTime=1731680095507; duration=0sec 2024-11-15T14:14:55,626 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:14:55,626 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:A 2024-11-15T14:14:55,626 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-15T14:14:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:55,627 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T14:14:55,627 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T14:14:55,627 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. because compaction request was cancelled 2024-11-15T14:14:55,627 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:C 2024-11-15T14:14:55,627 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:55,629 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:55,629 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/B is initiating minor compaction (all files) 2024-11-15T14:14:55,629 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/B in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:55,629 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/c281b18e4c2b406f82133a3a092b6506, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/7a79691256084b12b65f23b5fcd070a1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0d0249d90a9545cc8311fa0baa17408e] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=36.4 K 2024-11-15T14:14:55,630 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c281b18e4c2b406f82133a3a092b6506, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731680093414 2024-11-15T14:14:55,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741895_1071 (size=12301) 2024-11-15T14:14:55,632 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/ad5fe519b1d74e70b8df70ac8982460d 2024-11-15T14:14:55,633 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a79691256084b12b65f23b5fcd070a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731680094123 2024-11-15T14:14:55,634 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d0249d90a9545cc8311fa0baa17408e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731680094817 2024-11-15T14:14:55,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/56e34df3814c46159f9de3d0a6c106d3 is 50, key is test_row_0/B:col10/1731680095248/Put/seqid=0 2024-11-15T14:14:55,665 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#B#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:55,666 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/b7c8bd32ba2e4e40829b026288c89ceb is 50, key is test_row_0/B:col10/1731680095177/Put/seqid=0 2024-11-15T14:14:55,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741896_1072 (size=12301) 2024-11-15T14:14:55,677 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/56e34df3814c46159f9de3d0a6c106d3 2024-11-15T14:14:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T14:14:55,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741897_1073 (size=12949) 2024-11-15T14:14:55,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/69a0361ebf5a4ae18c30bda1ba89cdf6 is 50, key is test_row_0/C:col10/1731680095248/Put/seqid=0 2024-11-15T14:14:55,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741898_1074 (size=12301) 2024-11-15T14:14:55,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680155713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680155713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680155715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680155715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680155716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,739 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/8b7afb2cce8c4bf181bc8b1c81f85904 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/8b7afb2cce8c4bf181bc8b1c81f85904 2024-11-15T14:14:55,747 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 785aa89faccd90f1bafb73964185be68/C of 785aa89faccd90f1bafb73964185be68 into 8b7afb2cce8c4bf181bc8b1c81f85904(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:55,747 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:55,747 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/C, priority=12, startTime=1731680095094; duration=0sec 2024-11-15T14:14:55,747 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:55,747 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:C 2024-11-15T14:14:55,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680155832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680155832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680155832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680155832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680155832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:55,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:55,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680156043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680156043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680156043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680156043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680156044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,099 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/b7c8bd32ba2e4e40829b026288c89ceb as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b7c8bd32ba2e4e40829b026288c89ceb 2024-11-15T14:14:56,099 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/69a0361ebf5a4ae18c30bda1ba89cdf6 2024-11-15T14:14:56,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/ad5fe519b1d74e70b8df70ac8982460d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/ad5fe519b1d74e70b8df70ac8982460d 2024-11-15T14:14:56,123 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/B of 785aa89faccd90f1bafb73964185be68 into b7c8bd32ba2e4e40829b026288c89ceb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:56,123 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:56,123 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/B, priority=13, startTime=1731680095508; duration=0sec 2024-11-15T14:14:56,123 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/ad5fe519b1d74e70b8df70ac8982460d, entries=150, sequenceid=309, filesize=12.0 K 2024-11-15T14:14:56,123 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:56,123 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:B 2024-11-15T14:14:56,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/56e34df3814c46159f9de3d0a6c106d3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/56e34df3814c46159f9de3d0a6c106d3 2024-11-15T14:14:56,140 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/56e34df3814c46159f9de3d0a6c106d3, entries=150, sequenceid=309, filesize=12.0 K 2024-11-15T14:14:56,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/69a0361ebf5a4ae18c30bda1ba89cdf6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/69a0361ebf5a4ae18c30bda1ba89cdf6 2024-11-15T14:14:56,156 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/69a0361ebf5a4ae18c30bda1ba89cdf6, entries=150, sequenceid=309, filesize=12.0 K 2024-11-15T14:14:56,157 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 785aa89faccd90f1bafb73964185be68 in 628ms, sequenceid=309, compaction requested=true 2024-11-15T14:14:56,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:56,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:56,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-15T14:14:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-15T14:14:56,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T14:14:56,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1170 sec 2024-11-15T14:14:56,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1260 sec 2024-11-15T14:14:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T14:14:56,193 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:14:56,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:14:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-15T14:14:56,196 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:14:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T14:14:56,199 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:14:56,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:14:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T14:14:56,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T14:14:56,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:56,356 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T14:14:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:56,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:56,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:56,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:56,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:56,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:56,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/3d5462f6027c46f78e3cd3d49f633996 is 50, key is test_row_0/A:col10/1731680095714/Put/seqid=0 2024-11-15T14:14:56,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680156390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680156392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680156394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680156397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741899_1075 (size=12301) 2024-11-15T14:14:56,408 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/3d5462f6027c46f78e3cd3d49f633996 2024-11-15T14:14:56,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680156400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/7e3a16ddcb0e4679831feb8df22b02d5 is 50, key is test_row_0/B:col10/1731680095714/Put/seqid=0 2024-11-15T14:14:56,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741900_1076 (size=12301) 2024-11-15T14:14:56,475 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/7e3a16ddcb0e4679831feb8df22b02d5 2024-11-15T14:14:56,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/b12f3b54ed80410ebaf701c71d7c9ebc is 50, key is test_row_0/C:col10/1731680095714/Put/seqid=0 2024-11-15T14:14:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T14:14:56,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680156512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680156512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680156513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680156513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680156524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741901_1077 (size=12301) 2024-11-15T14:14:56,563 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/b12f3b54ed80410ebaf701c71d7c9ebc 2024-11-15T14:14:56,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/3d5462f6027c46f78e3cd3d49f633996 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/3d5462f6027c46f78e3cd3d49f633996 2024-11-15T14:14:56,586 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/3d5462f6027c46f78e3cd3d49f633996, entries=150, sequenceid=329, filesize=12.0 K 2024-11-15T14:14:56,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/7e3a16ddcb0e4679831feb8df22b02d5 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/7e3a16ddcb0e4679831feb8df22b02d5 2024-11-15T14:14:56,600 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/7e3a16ddcb0e4679831feb8df22b02d5, entries=150, sequenceid=329, filesize=12.0 K 2024-11-15T14:14:56,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/b12f3b54ed80410ebaf701c71d7c9ebc as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/b12f3b54ed80410ebaf701c71d7c9ebc 2024-11-15T14:14:56,608 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/b12f3b54ed80410ebaf701c71d7c9ebc, entries=150, sequenceid=329, filesize=12.0 K 2024-11-15T14:14:56,609 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 785aa89faccd90f1bafb73964185be68 in 253ms, sequenceid=329, compaction requested=true 2024-11-15T14:14:56,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:56,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:56,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-15T14:14:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-15T14:14:56,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-15T14:14:56,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 413 msec 2024-11-15T14:14:56,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 422 msec 2024-11-15T14:14:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:56,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T14:14:56,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:56,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:56,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:56,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:56,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:56,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:56,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/992a2fe7e206476a9e4a6673dd977ff0 is 50, key is test_row_0/A:col10/1731680096723/Put/seqid=0 2024-11-15T14:14:56,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741902_1078 (size=14741) 2024-11-15T14:14:56,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/992a2fe7e206476a9e4a6673dd977ff0 2024-11-15T14:14:56,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680156753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680156758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680156763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680156767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680156767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/d6e1d2d3ea8a489e82b797bc3e30311d is 50, key is test_row_0/B:col10/1731680096723/Put/seqid=0 2024-11-15T14:14:56,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741903_1079 (size=12301) 2024-11-15T14:14:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T14:14:56,822 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:14:56,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:14:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-15T14:14:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T14:14:56,830 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:14:56,831 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:14:56,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:14:56,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680156872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680156872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680156882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680156882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680156883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T14:14:56,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T14:14:56,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:56,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:56,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:56,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:56,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:57,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680157082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680157092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680157092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680157092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680157092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T14:14:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:57,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T14:14:57,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/d6e1d2d3ea8a489e82b797bc3e30311d 2024-11-15T14:14:57,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/5942a7bba433438ebc9bbce975680f01 is 50, key is test_row_0/C:col10/1731680096723/Put/seqid=0 2024-11-15T14:14:57,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741904_1080 (size=12301) 2024-11-15T14:14:57,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T14:14:57,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:57,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:57,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:57,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:57,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:57,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680157392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680157401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680157402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680157412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680157412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T14:14:57,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:57,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T14:14:57,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T14:14:57,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:57,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:57,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:57,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:57,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:57,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/5942a7bba433438ebc9bbce975680f01 2024-11-15T14:14:57,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/992a2fe7e206476a9e4a6673dd977ff0 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/992a2fe7e206476a9e4a6673dd977ff0 2024-11-15T14:14:57,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/992a2fe7e206476a9e4a6673dd977ff0, entries=200, sequenceid=345, filesize=14.4 K 2024-11-15T14:14:57,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/d6e1d2d3ea8a489e82b797bc3e30311d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d6e1d2d3ea8a489e82b797bc3e30311d 2024-11-15T14:14:57,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d6e1d2d3ea8a489e82b797bc3e30311d, entries=150, sequenceid=345, filesize=12.0 K 2024-11-15T14:14:57,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/5942a7bba433438ebc9bbce975680f01 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/5942a7bba433438ebc9bbce975680f01 2024-11-15T14:14:57,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/5942a7bba433438ebc9bbce975680f01, entries=150, sequenceid=345, filesize=12.0 K 2024-11-15T14:14:57,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 785aa89faccd90f1bafb73964185be68 in 925ms, sequenceid=345, compaction requested=true 2024-11-15T14:14:57,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:57,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:14:57,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:57,651 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:14:57,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:14:57,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:57,651 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:14:57,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:14:57,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:57,653 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:14:57,653 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/A is initiating minor compaction (all files) 2024-11-15T14:14:57,653 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/A in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:57,654 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f9a8f0fe6ffd42f9884bb182035c4ec4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/ad5fe519b1d74e70b8df70ac8982460d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/3d5462f6027c46f78e3cd3d49f633996, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/992a2fe7e206476a9e4a6673dd977ff0] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=51.1 K 2024-11-15T14:14:57,655 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9a8f0fe6ffd42f9884bb182035c4ec4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731680094817 2024-11-15T14:14:57,655 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:14:57,655 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad5fe519b1d74e70b8df70ac8982460d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731680095242 2024-11-15T14:14:57,656 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d5462f6027c46f78e3cd3d49f633996, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731680095648 2024-11-15T14:14:57,656 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 992a2fe7e206476a9e4a6673dd977ff0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731680096379 2024-11-15T14:14:57,659 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/B is initiating minor compaction (all files) 2024-11-15T14:14:57,659 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/B in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:57,659 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b7c8bd32ba2e4e40829b026288c89ceb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/56e34df3814c46159f9de3d0a6c106d3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/7e3a16ddcb0e4679831feb8df22b02d5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d6e1d2d3ea8a489e82b797bc3e30311d] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=48.7 K 2024-11-15T14:14:57,661 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting b7c8bd32ba2e4e40829b026288c89ceb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731680094817 2024-11-15T14:14:57,662 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 56e34df3814c46159f9de3d0a6c106d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731680095242 2024-11-15T14:14:57,662 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e3a16ddcb0e4679831feb8df22b02d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731680095648 2024-11-15T14:14:57,663 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting d6e1d2d3ea8a489e82b797bc3e30311d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731680096394 2024-11-15T14:14:57,679 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#A#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:57,680 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/92b277560ace4f20abe8e5d6bab11237 is 50, key is test_row_0/A:col10/1731680096723/Put/seqid=0 2024-11-15T14:14:57,683 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:57,684 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/0f1421f275494feabb771cf188731861 is 50, key is test_row_0/B:col10/1731680096723/Put/seqid=0 2024-11-15T14:14:57,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741905_1081 (size=13085) 2024-11-15T14:14:57,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741906_1082 (size=13085) 2024-11-15T14:14:57,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T14:14:57,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:57,756 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T14:14:57,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:57,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:57,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:57,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:57,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:57,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:57,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/5450a698751a4c66853f429e19a0da29 is 50, key is test_row_0/A:col10/1731680096762/Put/seqid=0 2024-11-15T14:14:57,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741907_1083 (size=12301) 2024-11-15T14:14:57,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:57,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680157928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680157929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680157930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680157930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680157932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:57,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:57,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T14:14:58,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680158042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680158042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680158042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680158042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680158052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,112 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/92b277560ace4f20abe8e5d6bab11237 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/92b277560ace4f20abe8e5d6bab11237 2024-11-15T14:14:58,120 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 785aa89faccd90f1bafb73964185be68/A of 785aa89faccd90f1bafb73964185be68 into 92b277560ace4f20abe8e5d6bab11237(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:58,120 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/0f1421f275494feabb771cf188731861 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0f1421f275494feabb771cf188731861 2024-11-15T14:14:58,120 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:58,120 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/A, priority=12, startTime=1731680097651; duration=0sec 2024-11-15T14:14:58,120 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:58,120 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:A 2024-11-15T14:14:58,120 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T14:14:58,123 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62085 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T14:14:58,123 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/C is initiating minor compaction (all files) 2024-11-15T14:14:58,124 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/C in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:58,124 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/8b7afb2cce8c4bf181bc8b1c81f85904, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fac5980c864440759781fb4675c33763, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/69a0361ebf5a4ae18c30bda1ba89cdf6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/b12f3b54ed80410ebaf701c71d7c9ebc, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/5942a7bba433438ebc9bbce975680f01] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=60.6 K 2024-11-15T14:14:58,125 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b7afb2cce8c4bf181bc8b1c81f85904, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731680094123 2024-11-15T14:14:58,125 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting fac5980c864440759781fb4675c33763, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731680094817 2024-11-15T14:14:58,126 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69a0361ebf5a4ae18c30bda1ba89cdf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731680095242 2024-11-15T14:14:58,126 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b12f3b54ed80410ebaf701c71d7c9ebc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731680095648 2024-11-15T14:14:58,127 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5942a7bba433438ebc9bbce975680f01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731680096394 2024-11-15T14:14:58,127 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 785aa89faccd90f1bafb73964185be68/B of 785aa89faccd90f1bafb73964185be68 into 0f1421f275494feabb771cf188731861(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:58,127 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:58,128 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/B, priority=12, startTime=1731680097651; duration=0sec 2024-11-15T14:14:58,128 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:58,128 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:B 2024-11-15T14:14:58,153 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#C#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:58,153 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/78c4a812bae6484fb1088ed540fe4e88 is 50, key is test_row_0/C:col10/1731680096723/Put/seqid=0 2024-11-15T14:14:58,175 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/5450a698751a4c66853f429e19a0da29 2024-11-15T14:14:58,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741908_1084 (size=13051) 2024-11-15T14:14:58,186 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/78c4a812bae6484fb1088ed540fe4e88 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/78c4a812bae6484fb1088ed540fe4e88 2024-11-15T14:14:58,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/63cfe2f46b884cea9b020312be9fe4c7 is 50, key is test_row_0/B:col10/1731680096762/Put/seqid=0 2024-11-15T14:14:58,199 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 785aa89faccd90f1bafb73964185be68/C of 785aa89faccd90f1bafb73964185be68 into 78c4a812bae6484fb1088ed540fe4e88(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:58,199 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:58,199 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/C, priority=11, startTime=1731680097651; duration=0sec 2024-11-15T14:14:58,200 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:58,200 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:C 2024-11-15T14:14:58,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741909_1085 (size=12301) 2024-11-15T14:14:58,207 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/63cfe2f46b884cea9b020312be9fe4c7 2024-11-15T14:14:58,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/9a461c71da274168a30c567ed720844b is 50, key is test_row_0/C:col10/1731680096762/Put/seqid=0 2024-11-15T14:14:58,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741910_1086 (size=12301) 2024-11-15T14:14:58,236 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/9a461c71da274168a30c567ed720844b 2024-11-15T14:14:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/5450a698751a4c66853f429e19a0da29 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/5450a698751a4c66853f429e19a0da29 2024-11-15T14:14:58,252 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/5450a698751a4c66853f429e19a0da29, entries=150, sequenceid=362, filesize=12.0 K 2024-11-15T14:14:58,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/63cfe2f46b884cea9b020312be9fe4c7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/63cfe2f46b884cea9b020312be9fe4c7 2024-11-15T14:14:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680158252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680158252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680158252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680158253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680158261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,264 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/63cfe2f46b884cea9b020312be9fe4c7, entries=150, sequenceid=362, filesize=12.0 K 2024-11-15T14:14:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/9a461c71da274168a30c567ed720844b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/9a461c71da274168a30c567ed720844b 2024-11-15T14:14:58,296 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/9a461c71da274168a30c567ed720844b, entries=150, sequenceid=362, filesize=12.0 K 2024-11-15T14:14:58,297 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 785aa89faccd90f1bafb73964185be68 in 542ms, sequenceid=362, compaction requested=false 2024-11-15T14:14:58,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:58,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:58,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-15T14:14:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-15T14:14:58,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-15T14:14:58,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4690 sec 2024-11-15T14:14:58,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.4820 sec 2024-11-15T14:14:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:58,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T14:14:58,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:58,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/b85be04b6c4e4a29a374452652102182 is 50, key is test_row_0/A:col10/1731680097929/Put/seqid=0 2024-11-15T14:14:58,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741911_1087 (size=12301) 2024-11-15T14:14:58,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/b85be04b6c4e4a29a374452652102182 2024-11-15T14:14:58,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680158583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/deb653a578d74f61a54a301676f4e037 is 50, key is test_row_0/B:col10/1731680097929/Put/seqid=0 2024-11-15T14:14:58,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680158586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680158589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680158591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680158591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741912_1088 (size=12301) 2024-11-15T14:14:58,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680158701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680158701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680158702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680158702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680158702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680158913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680158913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680158913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680158913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680158914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:58,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T14:14:58,992 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:14:58,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:14:58,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-15T14:14:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T14:14:58,994 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:14:58,995 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:14:58,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:14:59,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/deb653a578d74f61a54a301676f4e037 2024-11-15T14:14:59,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/351016014e8e4dc9a010a0bca532dab6 is 50, key is test_row_0/C:col10/1731680097929/Put/seqid=0 2024-11-15T14:14:59,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741913_1089 (size=12301) 2024-11-15T14:14:59,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T14:14:59,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T14:14:59,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:59,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:59,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:59,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:59,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:59,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680159222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680159222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680159223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680159222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680159223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T14:14:59,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:59,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:59,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:59,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:59,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:14:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T14:14:59,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/351016014e8e4dc9a010a0bca532dab6 2024-11-15T14:14:59,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/b85be04b6c4e4a29a374452652102182 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/b85be04b6c4e4a29a374452652102182 2024-11-15T14:14:59,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/b85be04b6c4e4a29a374452652102182, entries=150, sequenceid=381, filesize=12.0 K 2024-11-15T14:14:59,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/deb653a578d74f61a54a301676f4e037 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/deb653a578d74f61a54a301676f4e037 2024-11-15T14:14:59,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/deb653a578d74f61a54a301676f4e037, entries=150, sequenceid=381, filesize=12.0 K 2024-11-15T14:14:59,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/351016014e8e4dc9a010a0bca532dab6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/351016014e8e4dc9a010a0bca532dab6 2024-11-15T14:14:59,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/351016014e8e4dc9a010a0bca532dab6, entries=150, sequenceid=381, filesize=12.0 K 2024-11-15T14:14:59,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 785aa89faccd90f1bafb73964185be68 in 886ms, sequenceid=381, compaction requested=true 2024-11-15T14:14:59,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:59,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:14:59,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:59,453 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:59,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:14:59,453 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:59,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:59,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:14:59,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:59,455 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:59,455 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/A is initiating minor compaction (all files) 2024-11-15T14:14:59,455 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/A in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:59,455 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/92b277560ace4f20abe8e5d6bab11237, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/5450a698751a4c66853f429e19a0da29, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/b85be04b6c4e4a29a374452652102182] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=36.8 K 2024-11-15T14:14:59,456 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92b277560ace4f20abe8e5d6bab11237, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731680096394 2024-11-15T14:14:59,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T14:14:59,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:59,456 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:59,456 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/B is initiating minor compaction (all files) 2024-11-15T14:14:59,456 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T14:14:59,456 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/B in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:59,456 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0f1421f275494feabb771cf188731861, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/63cfe2f46b884cea9b020312be9fe4c7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/deb653a578d74f61a54a301676f4e037] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=36.8 K 2024-11-15T14:14:59,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:14:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:14:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:14:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:14:59,457 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f1421f275494feabb771cf188731861, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731680096394 2024-11-15T14:14:59,458 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5450a698751a4c66853f429e19a0da29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731680096747 2024-11-15T14:14:59,458 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 63cfe2f46b884cea9b020312be9fe4c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731680096747 2024-11-15T14:14:59,458 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b85be04b6c4e4a29a374452652102182, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731680097926 2024-11-15T14:14:59,459 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting deb653a578d74f61a54a301676f4e037, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731680097926 2024-11-15T14:14:59,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/361ffe3c15d843169cd8cc29dbb0b639 is 50, key is test_row_0/A:col10/1731680098588/Put/seqid=0 2024-11-15T14:14:59,479 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:59,480 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/7ad50191bc6d4d01af9f121568623cc3 is 50, key is test_row_0/A:col10/1731680097929/Put/seqid=0 2024-11-15T14:14:59,494 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:59,495 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/ee3dada5db344184a324030da43bbbe3 is 50, key is test_row_0/B:col10/1731680097929/Put/seqid=0 2024-11-15T14:14:59,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741915_1091 (size=13187) 2024-11-15T14:14:59,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741914_1090 (size=12301) 2024-11-15T14:14:59,508 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/7ad50191bc6d4d01af9f121568623cc3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/7ad50191bc6d4d01af9f121568623cc3 2024-11-15T14:14:59,516 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/A of 785aa89faccd90f1bafb73964185be68 into 7ad50191bc6d4d01af9f121568623cc3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:59,516 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:59,516 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/A, priority=13, startTime=1731680099453; duration=0sec 2024-11-15T14:14:59,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741916_1092 (size=13187) 2024-11-15T14:14:59,516 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:14:59,516 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:A 2024-11-15T14:14:59,517 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:14:59,521 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:14:59,521 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/C is initiating minor compaction (all files) 2024-11-15T14:14:59,521 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/C in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:14:59,521 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/78c4a812bae6484fb1088ed540fe4e88, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/9a461c71da274168a30c567ed720844b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/351016014e8e4dc9a010a0bca532dab6] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=36.8 K 2024-11-15T14:14:59,522 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78c4a812bae6484fb1088ed540fe4e88, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731680096394 2024-11-15T14:14:59,522 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a461c71da274168a30c567ed720844b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731680096747 2024-11-15T14:14:59,523 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 351016014e8e4dc9a010a0bca532dab6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731680097926 2024-11-15T14:14:59,528 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/ee3dada5db344184a324030da43bbbe3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/ee3dada5db344184a324030da43bbbe3 2024-11-15T14:14:59,536 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/B of 785aa89faccd90f1bafb73964185be68 into ee3dada5db344184a324030da43bbbe3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:59,536 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:59,536 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/B, priority=13, startTime=1731680099453; duration=0sec 2024-11-15T14:14:59,536 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:59,536 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:B 2024-11-15T14:14:59,539 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:14:59,540 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/a798e9c8ad87458f8caa23bfc87fc6fe is 50, key is test_row_0/C:col10/1731680097929/Put/seqid=0 2024-11-15T14:14:59,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741917_1093 (size=13153) 2024-11-15T14:14:59,583 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/a798e9c8ad87458f8caa23bfc87fc6fe as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/a798e9c8ad87458f8caa23bfc87fc6fe 2024-11-15T14:14:59,596 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/C of 785aa89faccd90f1bafb73964185be68 into a798e9c8ad87458f8caa23bfc87fc6fe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:14:59,596 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:14:59,596 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/C, priority=13, startTime=1731680099453; duration=0sec 2024-11-15T14:14:59,596 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:14:59,596 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:C 2024-11-15T14:14:59,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T14:14:59,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:14:59,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:14:59,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680159750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680159753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680159752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680159754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680159755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680159861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680159872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680159872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680159872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:14:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680159872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:14:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:14:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:14:59,902 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/361ffe3c15d843169cd8cc29dbb0b639 2024-11-15T14:14:59,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/a880e9b7b6364515a7b45dd0dbc86c72 is 50, key is test_row_0/B:col10/1731680098588/Put/seqid=0 2024-11-15T14:14:59,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741918_1094 (size=12301) 2024-11-15T14:15:00,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680160072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680160081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680160081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680160082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680160082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T14:15:00,317 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/a880e9b7b6364515a7b45dd0dbc86c72 2024-11-15T14:15:00,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/e7a47fcb87ad41fe8dc69991c08513eb is 50, key is test_row_0/C:col10/1731680098588/Put/seqid=0 2024-11-15T14:15:00,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741919_1095 (size=12301) 2024-11-15T14:15:00,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680160382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680160391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680160391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680160392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680160392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,732 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/e7a47fcb87ad41fe8dc69991c08513eb 2024-11-15T14:15:00,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/361ffe3c15d843169cd8cc29dbb0b639 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/361ffe3c15d843169cd8cc29dbb0b639 2024-11-15T14:15:00,744 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/361ffe3c15d843169cd8cc29dbb0b639, entries=150, sequenceid=398, filesize=12.0 K 2024-11-15T14:15:00,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/a880e9b7b6364515a7b45dd0dbc86c72 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/a880e9b7b6364515a7b45dd0dbc86c72 2024-11-15T14:15:00,751 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/a880e9b7b6364515a7b45dd0dbc86c72, entries=150, sequenceid=398, filesize=12.0 K 2024-11-15T14:15:00,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/e7a47fcb87ad41fe8dc69991c08513eb as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/e7a47fcb87ad41fe8dc69991c08513eb 2024-11-15T14:15:00,759 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/e7a47fcb87ad41fe8dc69991c08513eb, entries=150, sequenceid=398, filesize=12.0 K 2024-11-15T14:15:00,760 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 785aa89faccd90f1bafb73964185be68 in 1304ms, sequenceid=398, compaction requested=false 2024-11-15T14:15:00,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:15:00,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:00,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-15T14:15:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-15T14:15:00,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-15T14:15:00,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7670 sec 2024-11-15T14:15:00,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.7730 sec 2024-11-15T14:15:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:15:00,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T14:15:00,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:15:00,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:00,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:15:00,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:00,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:15:00,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:00,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/534aa8fae4ed4e939c792a58ec873b10 is 50, key is test_row_0/A:col10/1731680100893/Put/seqid=0 2024-11-15T14:15:00,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741920_1096 (size=12301) 2024-11-15T14:15:00,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/534aa8fae4ed4e939c792a58ec873b10 2024-11-15T14:15:00,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/61daf34473fa4cb0beaa0461c2813271 is 50, key is test_row_0/B:col10/1731680100893/Put/seqid=0 2024-11-15T14:15:00,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680160912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680160915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680160917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680160918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:00,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741921_1097 (size=12301) 2024-11-15T14:15:00,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680160923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680161032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680161032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680161032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680161032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680161041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T14:15:01,142 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:01,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-15T14:15:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T14:15:01,144 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:01,145 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:01,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:01,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680161241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680161241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680161242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680161242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T14:15:01,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680161252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T14:15:01,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:01,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:15:01,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:01,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:01,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:01,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/61daf34473fa4cb0beaa0461c2813271 2024-11-15T14:15:01,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/0b5eb81700b0496084d45d4a65521af8 is 50, key is test_row_0/C:col10/1731680100893/Put/seqid=0 2024-11-15T14:15:01,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741922_1098 (size=12301) 2024-11-15T14:15:01,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T14:15:01,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:01,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:15:01,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:01,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:01,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T14:15:01,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680161552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680161552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680161552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680161552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680161562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:01,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T14:15:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:15:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:01,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:01,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/0b5eb81700b0496084d45d4a65521af8 2024-11-15T14:15:01,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/534aa8fae4ed4e939c792a58ec873b10 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/534aa8fae4ed4e939c792a58ec873b10 2024-11-15T14:15:01,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/534aa8fae4ed4e939c792a58ec873b10, entries=150, sequenceid=417, filesize=12.0 K 2024-11-15T14:15:01,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T14:15:01,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:01,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:15:01,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:01,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:01,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:01,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/61daf34473fa4cb0beaa0461c2813271 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/61daf34473fa4cb0beaa0461c2813271 2024-11-15T14:15:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:01,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/61daf34473fa4cb0beaa0461c2813271, entries=150, sequenceid=417, filesize=12.0 K 2024-11-15T14:15:01,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/0b5eb81700b0496084d45d4a65521af8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/0b5eb81700b0496084d45d4a65521af8 2024-11-15T14:15:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T14:15:01,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/0b5eb81700b0496084d45d4a65521af8, entries=150, sequenceid=417, filesize=12.0 K 2024-11-15T14:15:01,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 785aa89faccd90f1bafb73964185be68 in 882ms, sequenceid=417, compaction requested=true 2024-11-15T14:15:01,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:15:01,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:01,777 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:01,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:01,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:01,778 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:01,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:01,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:01,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:01,779 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:01,779 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/B is initiating minor compaction (all files) 2024-11-15T14:15:01,779 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:01,779 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/B in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:01,779 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/A is initiating minor compaction (all files) 2024-11-15T14:15:01,779 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/A in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:01,779 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/ee3dada5db344184a324030da43bbbe3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/a880e9b7b6364515a7b45dd0dbc86c72, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/61daf34473fa4cb0beaa0461c2813271] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=36.9 K 2024-11-15T14:15:01,779 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/7ad50191bc6d4d01af9f121568623cc3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/361ffe3c15d843169cd8cc29dbb0b639, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/534aa8fae4ed4e939c792a58ec873b10] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=36.9 K 2024-11-15T14:15:01,780 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3dada5db344184a324030da43bbbe3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731680097926 2024-11-15T14:15:01,780 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ad50191bc6d4d01af9f121568623cc3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731680097926 2024-11-15T14:15:01,780 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting a880e9b7b6364515a7b45dd0dbc86c72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731680098581 2024-11-15T14:15:01,780 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 361ffe3c15d843169cd8cc29dbb0b639, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731680098581 2024-11-15T14:15:01,780 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 61daf34473fa4cb0beaa0461c2813271, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731680099751 2024-11-15T14:15:01,780 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 534aa8fae4ed4e939c792a58ec873b10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731680099751 2024-11-15T14:15:01,791 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#B#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:01,791 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/84f9fe3258024da8a272fec801e788c3 is 50, key is test_row_0/B:col10/1731680100893/Put/seqid=0 2024-11-15T14:15:01,792 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:01,793 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/5fa353f958d240c5b4cb05bad7c93284 is 50, key is test_row_0/A:col10/1731680100893/Put/seqid=0 2024-11-15T14:15:01,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741923_1099 (size=13289) 2024-11-15T14:15:01,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741924_1100 (size=13289) 2024-11-15T14:15:01,809 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/84f9fe3258024da8a272fec801e788c3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/84f9fe3258024da8a272fec801e788c3 2024-11-15T14:15:01,816 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/5fa353f958d240c5b4cb05bad7c93284 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/5fa353f958d240c5b4cb05bad7c93284 2024-11-15T14:15:01,819 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/B of 785aa89faccd90f1bafb73964185be68 into 84f9fe3258024da8a272fec801e788c3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:01,819 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:15:01,819 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/B, priority=13, startTime=1731680101778; duration=0sec 2024-11-15T14:15:01,819 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:01,819 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:B 2024-11-15T14:15:01,819 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:01,820 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:01,820 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/C is initiating minor compaction (all files) 2024-11-15T14:15:01,820 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/C in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:01,821 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/a798e9c8ad87458f8caa23bfc87fc6fe, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/e7a47fcb87ad41fe8dc69991c08513eb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/0b5eb81700b0496084d45d4a65521af8] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=36.9 K 2024-11-15T14:15:01,821 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting a798e9c8ad87458f8caa23bfc87fc6fe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731680097926 2024-11-15T14:15:01,822 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting e7a47fcb87ad41fe8dc69991c08513eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731680098581 2024-11-15T14:15:01,822 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b5eb81700b0496084d45d4a65521af8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731680099751 2024-11-15T14:15:01,825 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/A of 785aa89faccd90f1bafb73964185be68 into 5fa353f958d240c5b4cb05bad7c93284(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:01,825 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:15:01,825 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/A, priority=13, startTime=1731680101777; duration=0sec 2024-11-15T14:15:01,825 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:01,826 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:A 2024-11-15T14:15:01,833 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#C#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:01,833 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/3daf6276842e40a48e4754be5aab7912 is 50, key is test_row_0/C:col10/1731680100893/Put/seqid=0 2024-11-15T14:15:01,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741925_1101 (size=13255) 2024-11-15T14:15:01,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T14:15:01,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:01,912 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T14:15:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:15:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:15:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:15:01,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:01,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/dacfccb17fb748548dc792d2433e383f is 50, key is test_row_0/A:col10/1731680100910/Put/seqid=0 2024-11-15T14:15:01,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741926_1102 (size=12301) 2024-11-15T14:15:01,926 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/dacfccb17fb748548dc792d2433e383f 2024-11-15T14:15:01,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/42a9258faef24270a5dc38e976f90ab2 is 50, key is test_row_0/B:col10/1731680100910/Put/seqid=0 2024-11-15T14:15:01,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741927_1103 (size=12301) 2024-11-15T14:15:01,940 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/42a9258faef24270a5dc38e976f90ab2 2024-11-15T14:15:01,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/33a55e6b679a4a3f867c9e0a535432c3 is 50, key is test_row_0/C:col10/1731680100910/Put/seqid=0 2024-11-15T14:15:01,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741928_1104 (size=12301) 2024-11-15T14:15:01,956 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/33a55e6b679a4a3f867c9e0a535432c3 2024-11-15T14:15:01,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/dacfccb17fb748548dc792d2433e383f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/dacfccb17fb748548dc792d2433e383f 2024-11-15T14:15:01,970 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/dacfccb17fb748548dc792d2433e383f, entries=150, sequenceid=435, filesize=12.0 K 2024-11-15T14:15:01,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/42a9258faef24270a5dc38e976f90ab2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/42a9258faef24270a5dc38e976f90ab2 2024-11-15T14:15:01,978 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/42a9258faef24270a5dc38e976f90ab2, entries=150, sequenceid=435, filesize=12.0 K 2024-11-15T14:15:01,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/33a55e6b679a4a3f867c9e0a535432c3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/33a55e6b679a4a3f867c9e0a535432c3 2024-11-15T14:15:01,987 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/33a55e6b679a4a3f867c9e0a535432c3, entries=150, sequenceid=435, filesize=12.0 K 2024-11-15T14:15:01,988 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 785aa89faccd90f1bafb73964185be68 in 76ms, sequenceid=435, compaction requested=false 2024-11-15T14:15:01,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:15:01,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:01,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-15T14:15:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-15T14:15:01,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-15T14:15:01,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 845 msec 2024-11-15T14:15:01,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 851 msec 2024-11-15T14:15:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:15:02,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T14:15:02,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:15:02,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:02,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:15:02,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:02,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:15:02,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:02,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/e76ca34c7ed640b7b8cd94f410e46f25 is 50, key is test_row_0/A:col10/1731680102065/Put/seqid=0 2024-11-15T14:15:02,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741929_1105 (size=12301) 2024-11-15T14:15:02,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680162139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680162139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731680102136, max timestamp=1731680102140Num uniques 859; 2024-11-15T14:15:02,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T14:15:02,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731680102136, max timestamp=1731680102140Num uniques 859; 2024-11-15T14:15:02,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T14:15:02,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731680102136, max timestamp=1731680102140Num uniques 859; 2024-11-15T14:15:02,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T14:15:02,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680162168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680162178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680162179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,244 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/3daf6276842e40a48e4754be5aab7912 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/3daf6276842e40a48e4754be5aab7912 2024-11-15T14:15:02,250 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/C of 785aa89faccd90f1bafb73964185be68 into 3daf6276842e40a48e4754be5aab7912(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:02,250 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:15:02,250 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/C, priority=13, startTime=1731680101778; duration=0sec 2024-11-15T14:15:02,250 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:02,250 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:C 2024-11-15T14:15:02,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680162252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680162252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T14:15:02,282 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:02,282 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-15T14:15:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T14:15:02,284 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:02,285 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:02,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:02,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680162281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680162282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680162282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T14:15:02,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T14:15:02,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:02,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:15:02,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:02,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:02,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:02,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:02,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680162462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680162462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680162492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680162492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680162492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/e76ca34c7ed640b7b8cd94f410e46f25 2024-11-15T14:15:02,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/fd6b9aa4f0a843adabd64246b7588d04 is 50, key is test_row_0/B:col10/1731680102065/Put/seqid=0 2024-11-15T14:15:02,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741930_1106 (size=12301) 2024-11-15T14:15:02,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T14:15:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:15:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:02,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:02,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T14:15:02,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T14:15:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:15:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:02,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:02,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680162771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680162771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680162802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680162802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680162802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:02,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:02,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:02,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T14:15:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:15:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:02,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T14:15:02,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/fd6b9aa4f0a843adabd64246b7588d04 2024-11-15T14:15:02,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/f439c93b0bb94431a4091a9c529981f9 is 50, key is test_row_0/C:col10/1731680102065/Put/seqid=0 2024-11-15T14:15:02,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741931_1107 (size=12301) 2024-11-15T14:15:02,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/f439c93b0bb94431a4091a9c529981f9 2024-11-15T14:15:02,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/e76ca34c7ed640b7b8cd94f410e46f25 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/e76ca34c7ed640b7b8cd94f410e46f25 2024-11-15T14:15:02,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/e76ca34c7ed640b7b8cd94f410e46f25, entries=150, sequenceid=446, filesize=12.0 K 2024-11-15T14:15:02,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/fd6b9aa4f0a843adabd64246b7588d04 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/fd6b9aa4f0a843adabd64246b7588d04 2024-11-15T14:15:02,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/fd6b9aa4f0a843adabd64246b7588d04, entries=150, sequenceid=446, filesize=12.0 K 2024-11-15T14:15:02,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/f439c93b0bb94431a4091a9c529981f9 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/f439c93b0bb94431a4091a9c529981f9 2024-11-15T14:15:02,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/f439c93b0bb94431a4091a9c529981f9, entries=150, sequenceid=446, filesize=12.0 K 2024-11-15T14:15:02,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 785aa89faccd90f1bafb73964185be68 in 872ms, sequenceid=446, compaction requested=true 2024-11-15T14:15:02,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:15:02,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:02,981 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:02,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:02,982 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:02,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:02,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:02,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:02,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:02,983 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:02,983 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:02,983 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/B is initiating minor compaction (all files) 2024-11-15T14:15:02,983 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/A is initiating minor compaction (all files) 2024-11-15T14:15:02,983 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/B in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:02,983 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/A in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:02,983 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/84f9fe3258024da8a272fec801e788c3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/42a9258faef24270a5dc38e976f90ab2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/fd6b9aa4f0a843adabd64246b7588d04] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=37.0 K 2024-11-15T14:15:02,983 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/5fa353f958d240c5b4cb05bad7c93284, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/dacfccb17fb748548dc792d2433e383f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/e76ca34c7ed640b7b8cd94f410e46f25] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=37.0 K 2024-11-15T14:15:02,984 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 84f9fe3258024da8a272fec801e788c3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731680099751 2024-11-15T14:15:02,984 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fa353f958d240c5b4cb05bad7c93284, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731680099751 2024-11-15T14:15:02,984 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting dacfccb17fb748548dc792d2433e383f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731680100910 2024-11-15T14:15:02,984 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 42a9258faef24270a5dc38e976f90ab2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731680100910 2024-11-15T14:15:02,985 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e76ca34c7ed640b7b8cd94f410e46f25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731680102065 2024-11-15T14:15:02,985 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting fd6b9aa4f0a843adabd64246b7588d04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731680102065 2024-11-15T14:15:02,993 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#A#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:02,993 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#B#compaction#95 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:02,994 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/5f39a592e1d0406984bf28d160770bf0 is 50, key is test_row_0/B:col10/1731680102065/Put/seqid=0 2024-11-15T14:15:02,994 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/bf779987082b495dab9fc85cc66818f2 is 50, key is test_row_0/A:col10/1731680102065/Put/seqid=0 2024-11-15T14:15:03,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741932_1108 (size=13391) 2024-11-15T14:15:03,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741933_1109 (size=13391) 2024-11-15T14:15:03,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T14:15:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:03,051 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-15T14:15:03,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:15:03,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:03,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:15:03,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:03,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:15:03,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:03,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/98a27ea4d63546c79545bcff07efb159 is 50, key is test_row_0/A:col10/1731680102176/Put/seqid=0 2024-11-15T14:15:03,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741934_1110 (size=12301) 2024-11-15T14:15:03,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. as already flushing 2024-11-15T14:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:15:03,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680163298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:03,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:03,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680163302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:03,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:03,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35954 deadline: 1731680163312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:03,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:03,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35972 deadline: 1731680163312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:03,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35996 deadline: 1731680163312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:03,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36028 deadline: 1731680163411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:03,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:03,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35988 deadline: 1731680163412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:03,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:03,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:03,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:03,416 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/5f39a592e1d0406984bf28d160770bf0 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/5f39a592e1d0406984bf28d160770bf0 2024-11-15T14:15:03,416 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/bf779987082b495dab9fc85cc66818f2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/bf779987082b495dab9fc85cc66818f2 2024-11-15T14:15:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T14:15:03,423 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/A of 785aa89faccd90f1bafb73964185be68 into bf779987082b495dab9fc85cc66818f2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:03,423 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/B of 785aa89faccd90f1bafb73964185be68 into 5f39a592e1d0406984bf28d160770bf0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:03,423 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:15:03,423 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:15:03,423 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/A, priority=13, startTime=1731680102981; duration=0sec 2024-11-15T14:15:03,423 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/B, priority=13, startTime=1731680102981; duration=0sec 2024-11-15T14:15:03,423 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:03,423 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:B 2024-11-15T14:15:03,423 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:03,423 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:A 2024-11-15T14:15:03,423 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:03,425 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:03,425 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 785aa89faccd90f1bafb73964185be68/C is initiating minor compaction (all files) 2024-11-15T14:15:03,425 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 785aa89faccd90f1bafb73964185be68/C in TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:03,425 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/3daf6276842e40a48e4754be5aab7912, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/33a55e6b679a4a3f867c9e0a535432c3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/f439c93b0bb94431a4091a9c529981f9] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp, totalSize=37.0 K 2024-11-15T14:15:03,426 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 3daf6276842e40a48e4754be5aab7912, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731680099751 2024-11-15T14:15:03,426 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 33a55e6b679a4a3f867c9e0a535432c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731680100910 2024-11-15T14:15:03,427 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting f439c93b0bb94431a4091a9c529981f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731680102065 2024-11-15T14:15:03,437 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 785aa89faccd90f1bafb73964185be68#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:03,438 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/7207c3acd24147bbbc4c13bd85dac148 is 50, key is test_row_0/C:col10/1731680102065/Put/seqid=0 2024-11-15T14:15:03,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741935_1111 (size=13357) 2024-11-15T14:15:03,448 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/7207c3acd24147bbbc4c13bd85dac148 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/7207c3acd24147bbbc4c13bd85dac148 2024-11-15T14:15:03,454 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 785aa89faccd90f1bafb73964185be68/C of 785aa89faccd90f1bafb73964185be68 into 7207c3acd24147bbbc4c13bd85dac148(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:03,455 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:15:03,455 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., storeName=785aa89faccd90f1bafb73964185be68/C, priority=13, startTime=1731680102982; duration=0sec 2024-11-15T14:15:03,455 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:03,455 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:C 2024-11-15T14:15:03,462 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/98a27ea4d63546c79545bcff07efb159 2024-11-15T14:15:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/a09c488045c04c5e8c0e096e26532763 is 50, key is test_row_0/B:col10/1731680102176/Put/seqid=0 2024-11-15T14:15:03,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741936_1112 (size=12301) 2024-11-15T14:15:03,475 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/a09c488045c04c5e8c0e096e26532763 2024-11-15T14:15:03,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/fdf3bcceb55c4e7088e85c78518507f4 is 50, key is test_row_0/C:col10/1731680102176/Put/seqid=0 2024-11-15T14:15:03,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741937_1113 (size=12301) 2024-11-15T14:15:03,489 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/fdf3bcceb55c4e7088e85c78518507f4 2024-11-15T14:15:03,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/98a27ea4d63546c79545bcff07efb159 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/98a27ea4d63546c79545bcff07efb159 2024-11-15T14:15:03,504 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/98a27ea4d63546c79545bcff07efb159, entries=150, sequenceid=481, filesize=12.0 K 2024-11-15T14:15:03,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/a09c488045c04c5e8c0e096e26532763 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/a09c488045c04c5e8c0e096e26532763 2024-11-15T14:15:03,512 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/a09c488045c04c5e8c0e096e26532763, entries=150, sequenceid=481, filesize=12.0 K 2024-11-15T14:15:03,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/fdf3bcceb55c4e7088e85c78518507f4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fdf3bcceb55c4e7088e85c78518507f4 2024-11-15T14:15:03,526 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fdf3bcceb55c4e7088e85c78518507f4, entries=150, sequenceid=481, filesize=12.0 K 2024-11-15T14:15:03,528 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 785aa89faccd90f1bafb73964185be68 in 478ms, sequenceid=481, compaction requested=false 2024-11-15T14:15:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:15:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-15T14:15:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-15T14:15:03,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-15T14:15:03,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2460 sec 2024-11-15T14:15:03,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.2570 sec 2024-11-15T14:15:03,547 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-15T14:15:03,547 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-15T14:15:03,547 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:03,547 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:03,550 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-15T14:15:03,550 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:03,550 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-15T14:15:03,550 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:03,551 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:03,552 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:03,552 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:03,552 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:03,552 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:03,552 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:03,552 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:03,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:03,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:03,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:03,552 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:03,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 785aa89faccd90f1bafb73964185be68 2024-11-15T14:15:03,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T14:15:03,624 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-15T14:15:03,624 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-15T14:15:03,624 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:03,624 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:03,624 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:03,624 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:03,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:15:03,624 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:03,624 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:03,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:03,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:03,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:03,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:15:03,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:03,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:15:03,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:03,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/1e8ec721f80c42858dd4c32d2d7be415 is 50, key is test_row_0/A:col10/1731680103622/Put/seqid=0 2024-11-15T14:15:03,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741938_1114 (size=12301) 2024-11-15T14:15:04,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/1e8ec721f80c42858dd4c32d2d7be415 2024-11-15T14:15:04,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/f09ca7de91d848eeb2c7ca7dbb5a6cb7 is 50, key is test_row_0/B:col10/1731680103622/Put/seqid=0 2024-11-15T14:15:04,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741939_1115 (size=12301) 2024-11-15T14:15:04,324 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-15T14:15:04,324 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:04,324 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:04,324 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:04,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:04,333 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-15T14:15:04,333 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:04,333 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:04,333 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:04,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:04,344 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-15T14:15:04,344 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:04,344 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:04,344 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:04,344 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T14:15:04,433 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:04,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T14:15:04,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-15T14:15:04,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-15T14:15:04,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-15T14:15:04,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-15T14:15:04,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-15T14:15:04,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T14:15:04,434 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1964 2024-11-15T14:15:04,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1932 2024-11-15T14:15:04,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T14:15:04,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 904 2024-11-15T14:15:04,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2708 rows 2024-11-15T14:15:04,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 909 2024-11-15T14:15:04,435 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2726 rows 2024-11-15T14:15:04,435 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T14:15:04,436 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T14:15:04,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:04,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:04,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:04,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T14:15:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T14:15:04,451 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680104451"}]},"ts":"1731680104451"} 2024-11-15T14:15:04,453 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T14:15:04,454 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T14:15:04,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T14:15:04,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=785aa89faccd90f1bafb73964185be68, UNASSIGN}] 2024-11-15T14:15:04,461 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=785aa89faccd90f1bafb73964185be68, UNASSIGN 2024-11-15T14:15:04,462 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=785aa89faccd90f1bafb73964185be68, regionState=CLOSING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:15:04,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=785aa89faccd90f1bafb73964185be68, UNASSIGN because future has completed 2024-11-15T14:15:04,465 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T14:15:04,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:15:04,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/f09ca7de91d848eeb2c7ca7dbb5a6cb7 2024-11-15T14:15:04,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/fa131a91b1764b538bd98b9cc86d07e3 is 50, key is test_row_0/C:col10/1731680103622/Put/seqid=0 2024-11-15T14:15:04,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741940_1116 (size=12301) 2024-11-15T14:15:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T14:15:04,625 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 785aa89faccd90f1bafb73964185be68 2024-11-15T14:15:04,626 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T14:15:04,627 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 785aa89faccd90f1bafb73964185be68, disabling compactions & flushes 2024-11-15T14:15:04,627 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:04,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T14:15:04,811 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T14:15:04,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/fa131a91b1764b538bd98b9cc86d07e3 2024-11-15T14:15:04,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/1e8ec721f80c42858dd4c32d2d7be415 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/1e8ec721f80c42858dd4c32d2d7be415 2024-11-15T14:15:04,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/1e8ec721f80c42858dd4c32d2d7be415, entries=150, sequenceid=497, filesize=12.0 K 2024-11-15T14:15:04,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/f09ca7de91d848eeb2c7ca7dbb5a6cb7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/f09ca7de91d848eeb2c7ca7dbb5a6cb7 2024-11-15T14:15:04,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/f09ca7de91d848eeb2c7ca7dbb5a6cb7, entries=150, sequenceid=497, filesize=12.0 K 2024-11-15T14:15:04,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/fa131a91b1764b538bd98b9cc86d07e3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fa131a91b1764b538bd98b9cc86d07e3 2024-11-15T14:15:04,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fa131a91b1764b538bd98b9cc86d07e3, entries=150, sequenceid=497, filesize=12.0 K 2024-11-15T14:15:04,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=20.13 KB/20610 for 785aa89faccd90f1bafb73964185be68 in 1338ms, sequenceid=497, compaction requested=true 2024-11-15T14:15:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 785aa89faccd90f1bafb73964185be68: 2024-11-15T14:15:04,963 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:04,963 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:04,963 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. because compaction request was cancelled 2024-11-15T14:15:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:04,963 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. after waiting 0 ms 2024-11-15T14:15:04,963 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:A 2024-11-15T14:15:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:04,963 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. because compaction request was cancelled 2024-11-15T14:15:04,963 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 785aa89faccd90f1bafb73964185be68:C, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:04,963 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:B 2024-11-15T14:15:04,963 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. because compaction request was cancelled 2024-11-15T14:15:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:04,963 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 785aa89faccd90f1bafb73964185be68:C 2024-11-15T14:15:04,963 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 785aa89faccd90f1bafb73964185be68 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-15T14:15:04,963 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=A 2024-11-15T14:15:04,963 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:04,963 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=B 2024-11-15T14:15:04,963 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:04,963 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 785aa89faccd90f1bafb73964185be68, store=C 2024-11-15T14:15:04,964 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:04,967 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/15a69e66c6d14dec8008a4302120f558 is 50, key is test_row_0/A:col10/1731680104342/Put/seqid=0 2024-11-15T14:15:04,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741941_1117 (size=9857) 2024-11-15T14:15:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T14:15:05,373 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/15a69e66c6d14dec8008a4302120f558 2024-11-15T14:15:05,385 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/d71854e68a2b44a68b6af3f99e5e10d1 is 50, key is test_row_0/B:col10/1731680104342/Put/seqid=0 2024-11-15T14:15:05,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741942_1118 (size=9857) 2024-11-15T14:15:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T14:15:05,791 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/d71854e68a2b44a68b6af3f99e5e10d1 2024-11-15T14:15:05,805 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/79c5b43a663a4dd7805bab06016fcf12 is 50, key is test_row_0/C:col10/1731680104342/Put/seqid=0 2024-11-15T14:15:05,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741943_1119 (size=9857) 2024-11-15T14:15:06,210 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/79c5b43a663a4dd7805bab06016fcf12 2024-11-15T14:15:06,218 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/A/15a69e66c6d14dec8008a4302120f558 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/15a69e66c6d14dec8008a4302120f558 2024-11-15T14:15:06,224 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/15a69e66c6d14dec8008a4302120f558, entries=100, sequenceid=503, filesize=9.6 K 2024-11-15T14:15:06,225 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/B/d71854e68a2b44a68b6af3f99e5e10d1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d71854e68a2b44a68b6af3f99e5e10d1 2024-11-15T14:15:06,229 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d71854e68a2b44a68b6af3f99e5e10d1, entries=100, sequenceid=503, filesize=9.6 K 2024-11-15T14:15:06,230 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/.tmp/C/79c5b43a663a4dd7805bab06016fcf12 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/79c5b43a663a4dd7805bab06016fcf12 2024-11-15T14:15:06,234 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/79c5b43a663a4dd7805bab06016fcf12, entries=100, sequenceid=503, filesize=9.6 K 2024-11-15T14:15:06,235 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 785aa89faccd90f1bafb73964185be68 in 1272ms, sequenceid=503, compaction requested=true 2024-11-15T14:15:06,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/3388db958ff6440c8f0809727efcad5c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f5e8a24474f14395944f2895ed7d22f8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/9b83c565e5c144ac9b23cc87cccb6f80, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/720d86a998f04e6d948dfce96409cde0, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/2de211160af14e23bf93a2e510c00c12, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/264a6940dd7a4b9aaca2df2b7c0f88e7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/2e65ac0ab86e45d8a1c2527979418555, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/8f8e47d17b78473bb215ad031aba3b02, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/dddc2ad8482c48658bb1b76cfc424e6b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/cb34c96bca1e4d57ac3f13a9fe858695, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/6d0a08c1ddec4a99bf42203e6845b572, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/24233f863197493ea465e70803ab3220, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f70ec0bf132e4f53ae1d677c81532847, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/703f1556c47542049469e8f4ea4f7564, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/072658d744a9464fb547b64d76b19244, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f38e14d47b1b4a53b72e07bfc4f141d8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f3edf02cd9cf4f7e8aaec7a6e3626075, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/ece81dd4bc5f4483a6677388e681a842, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f9a8f0fe6ffd42f9884bb182035c4ec4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/1360819225d54047b263455d8324460d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/ad5fe519b1d74e70b8df70ac8982460d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/3d5462f6027c46f78e3cd3d49f633996, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/992a2fe7e206476a9e4a6673dd977ff0, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/92b277560ace4f20abe8e5d6bab11237, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/5450a698751a4c66853f429e19a0da29, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/7ad50191bc6d4d01af9f121568623cc3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/b85be04b6c4e4a29a374452652102182, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/361ffe3c15d843169cd8cc29dbb0b639, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/5fa353f958d240c5b4cb05bad7c93284, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/534aa8fae4ed4e939c792a58ec873b10, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/dacfccb17fb748548dc792d2433e383f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/e76ca34c7ed640b7b8cd94f410e46f25] to archive 2024-11-15T14:15:06,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:15:06,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/3388db958ff6440c8f0809727efcad5c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/3388db958ff6440c8f0809727efcad5c 2024-11-15T14:15:06,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f5e8a24474f14395944f2895ed7d22f8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f5e8a24474f14395944f2895ed7d22f8 2024-11-15T14:15:06,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/9b83c565e5c144ac9b23cc87cccb6f80 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/9b83c565e5c144ac9b23cc87cccb6f80 2024-11-15T14:15:06,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/720d86a998f04e6d948dfce96409cde0 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/720d86a998f04e6d948dfce96409cde0 2024-11-15T14:15:06,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/2de211160af14e23bf93a2e510c00c12 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/2de211160af14e23bf93a2e510c00c12 2024-11-15T14:15:06,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/264a6940dd7a4b9aaca2df2b7c0f88e7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/264a6940dd7a4b9aaca2df2b7c0f88e7 2024-11-15T14:15:06,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/2e65ac0ab86e45d8a1c2527979418555 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/2e65ac0ab86e45d8a1c2527979418555 2024-11-15T14:15:06,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/8f8e47d17b78473bb215ad031aba3b02 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/8f8e47d17b78473bb215ad031aba3b02 2024-11-15T14:15:06,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/dddc2ad8482c48658bb1b76cfc424e6b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/dddc2ad8482c48658bb1b76cfc424e6b 2024-11-15T14:15:06,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/cb34c96bca1e4d57ac3f13a9fe858695 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/cb34c96bca1e4d57ac3f13a9fe858695 2024-11-15T14:15:06,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/6d0a08c1ddec4a99bf42203e6845b572 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/6d0a08c1ddec4a99bf42203e6845b572 2024-11-15T14:15:06,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/24233f863197493ea465e70803ab3220 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/24233f863197493ea465e70803ab3220 2024-11-15T14:15:06,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f70ec0bf132e4f53ae1d677c81532847 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f70ec0bf132e4f53ae1d677c81532847 2024-11-15T14:15:06,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/703f1556c47542049469e8f4ea4f7564 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/703f1556c47542049469e8f4ea4f7564 2024-11-15T14:15:06,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/072658d744a9464fb547b64d76b19244 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/072658d744a9464fb547b64d76b19244 2024-11-15T14:15:06,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f38e14d47b1b4a53b72e07bfc4f141d8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f38e14d47b1b4a53b72e07bfc4f141d8 2024-11-15T14:15:06,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f3edf02cd9cf4f7e8aaec7a6e3626075 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f3edf02cd9cf4f7e8aaec7a6e3626075 2024-11-15T14:15:06,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/ece81dd4bc5f4483a6677388e681a842 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/ece81dd4bc5f4483a6677388e681a842 2024-11-15T14:15:06,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f9a8f0fe6ffd42f9884bb182035c4ec4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/f9a8f0fe6ffd42f9884bb182035c4ec4 2024-11-15T14:15:06,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/1360819225d54047b263455d8324460d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/1360819225d54047b263455d8324460d 2024-11-15T14:15:06,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/ad5fe519b1d74e70b8df70ac8982460d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/ad5fe519b1d74e70b8df70ac8982460d 2024-11-15T14:15:06,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/3d5462f6027c46f78e3cd3d49f633996 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/3d5462f6027c46f78e3cd3d49f633996 2024-11-15T14:15:06,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/992a2fe7e206476a9e4a6673dd977ff0 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/992a2fe7e206476a9e4a6673dd977ff0 2024-11-15T14:15:06,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/92b277560ace4f20abe8e5d6bab11237 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/92b277560ace4f20abe8e5d6bab11237 2024-11-15T14:15:06,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/5450a698751a4c66853f429e19a0da29 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/5450a698751a4c66853f429e19a0da29 2024-11-15T14:15:06,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/7ad50191bc6d4d01af9f121568623cc3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/7ad50191bc6d4d01af9f121568623cc3 2024-11-15T14:15:06,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/b85be04b6c4e4a29a374452652102182 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/b85be04b6c4e4a29a374452652102182 2024-11-15T14:15:06,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/361ffe3c15d843169cd8cc29dbb0b639 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/361ffe3c15d843169cd8cc29dbb0b639 2024-11-15T14:15:06,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/5fa353f958d240c5b4cb05bad7c93284 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/5fa353f958d240c5b4cb05bad7c93284 2024-11-15T14:15:06,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/534aa8fae4ed4e939c792a58ec873b10 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/534aa8fae4ed4e939c792a58ec873b10 2024-11-15T14:15:06,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/dacfccb17fb748548dc792d2433e383f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/dacfccb17fb748548dc792d2433e383f 2024-11-15T14:15:06,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/e76ca34c7ed640b7b8cd94f410e46f25 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/e76ca34c7ed640b7b8cd94f410e46f25 2024-11-15T14:15:06,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d51c9ea99b5c49f9ad521055e7433ff2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0be0ffb4d10e4ccaadce5c48cd8a951b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0a5a7b75975e444c9dbc8106de536b0b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b55713844aa545289aad03a16a4f1458, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/afc71d668014404f90ceacc0c13e8433, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/4a098406aedc402cb726429e82ce79a7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/aa330313eb3c4b38bda31834edecae55, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/be2503f0d2514652ae1b4926a8a5b219, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/93d4d14ab5ce42e0b712fed8d7adb995, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/5194dc1a29c94dc592f124c2383e9786, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/75e66dd8de9145108929385c45f12a04, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/4a8db4d13b6b4b19826607ae853aad28, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/e6e5bec6e9da402794a38111eb9c0945, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b3fadf8a5610405b994a08ea85da1b0f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/1a29e60500ef4b95a4786531ad72e2c8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/c281b18e4c2b406f82133a3a092b6506, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/897caf6eeec34761885f0b83b538381d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/7a79691256084b12b65f23b5fcd070a1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b7c8bd32ba2e4e40829b026288c89ceb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0d0249d90a9545cc8311fa0baa17408e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/56e34df3814c46159f9de3d0a6c106d3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/7e3a16ddcb0e4679831feb8df22b02d5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0f1421f275494feabb771cf188731861, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d6e1d2d3ea8a489e82b797bc3e30311d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/63cfe2f46b884cea9b020312be9fe4c7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/ee3dada5db344184a324030da43bbbe3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/deb653a578d74f61a54a301676f4e037, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/a880e9b7b6364515a7b45dd0dbc86c72, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/84f9fe3258024da8a272fec801e788c3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/61daf34473fa4cb0beaa0461c2813271, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/42a9258faef24270a5dc38e976f90ab2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/fd6b9aa4f0a843adabd64246b7588d04] to archive 2024-11-15T14:15:06,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:15:06,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d51c9ea99b5c49f9ad521055e7433ff2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d51c9ea99b5c49f9ad521055e7433ff2 2024-11-15T14:15:06,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0be0ffb4d10e4ccaadce5c48cd8a951b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0be0ffb4d10e4ccaadce5c48cd8a951b 2024-11-15T14:15:06,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0a5a7b75975e444c9dbc8106de536b0b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0a5a7b75975e444c9dbc8106de536b0b 2024-11-15T14:15:06,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b55713844aa545289aad03a16a4f1458 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b55713844aa545289aad03a16a4f1458 2024-11-15T14:15:06,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/afc71d668014404f90ceacc0c13e8433 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/afc71d668014404f90ceacc0c13e8433 2024-11-15T14:15:06,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/4a098406aedc402cb726429e82ce79a7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/4a098406aedc402cb726429e82ce79a7 2024-11-15T14:15:06,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/aa330313eb3c4b38bda31834edecae55 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/aa330313eb3c4b38bda31834edecae55 2024-11-15T14:15:06,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/be2503f0d2514652ae1b4926a8a5b219 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/be2503f0d2514652ae1b4926a8a5b219 2024-11-15T14:15:06,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/93d4d14ab5ce42e0b712fed8d7adb995 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/93d4d14ab5ce42e0b712fed8d7adb995 2024-11-15T14:15:06,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/5194dc1a29c94dc592f124c2383e9786 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/5194dc1a29c94dc592f124c2383e9786 2024-11-15T14:15:06,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/75e66dd8de9145108929385c45f12a04 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/75e66dd8de9145108929385c45f12a04 2024-11-15T14:15:06,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/4a8db4d13b6b4b19826607ae853aad28 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/4a8db4d13b6b4b19826607ae853aad28 2024-11-15T14:15:06,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/e6e5bec6e9da402794a38111eb9c0945 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/e6e5bec6e9da402794a38111eb9c0945 2024-11-15T14:15:06,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b3fadf8a5610405b994a08ea85da1b0f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b3fadf8a5610405b994a08ea85da1b0f 2024-11-15T14:15:06,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/1a29e60500ef4b95a4786531ad72e2c8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/1a29e60500ef4b95a4786531ad72e2c8 2024-11-15T14:15:06,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/c281b18e4c2b406f82133a3a092b6506 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/c281b18e4c2b406f82133a3a092b6506 2024-11-15T14:15:06,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/897caf6eeec34761885f0b83b538381d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/897caf6eeec34761885f0b83b538381d 2024-11-15T14:15:06,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/7a79691256084b12b65f23b5fcd070a1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/7a79691256084b12b65f23b5fcd070a1 2024-11-15T14:15:06,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b7c8bd32ba2e4e40829b026288c89ceb to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/b7c8bd32ba2e4e40829b026288c89ceb 2024-11-15T14:15:06,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0d0249d90a9545cc8311fa0baa17408e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0d0249d90a9545cc8311fa0baa17408e 2024-11-15T14:15:06,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/56e34df3814c46159f9de3d0a6c106d3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/56e34df3814c46159f9de3d0a6c106d3 2024-11-15T14:15:06,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/7e3a16ddcb0e4679831feb8df22b02d5 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/7e3a16ddcb0e4679831feb8df22b02d5 2024-11-15T14:15:06,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0f1421f275494feabb771cf188731861 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/0f1421f275494feabb771cf188731861 2024-11-15T14:15:06,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d6e1d2d3ea8a489e82b797bc3e30311d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d6e1d2d3ea8a489e82b797bc3e30311d 2024-11-15T14:15:06,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/63cfe2f46b884cea9b020312be9fe4c7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/63cfe2f46b884cea9b020312be9fe4c7 2024-11-15T14:15:06,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/ee3dada5db344184a324030da43bbbe3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/ee3dada5db344184a324030da43bbbe3 2024-11-15T14:15:06,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/deb653a578d74f61a54a301676f4e037 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/deb653a578d74f61a54a301676f4e037 2024-11-15T14:15:06,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/a880e9b7b6364515a7b45dd0dbc86c72 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/a880e9b7b6364515a7b45dd0dbc86c72 2024-11-15T14:15:06,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/84f9fe3258024da8a272fec801e788c3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/84f9fe3258024da8a272fec801e788c3 2024-11-15T14:15:06,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/61daf34473fa4cb0beaa0461c2813271 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/61daf34473fa4cb0beaa0461c2813271 2024-11-15T14:15:06,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/42a9258faef24270a5dc38e976f90ab2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/42a9258faef24270a5dc38e976f90ab2 2024-11-15T14:15:06,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/fd6b9aa4f0a843adabd64246b7588d04 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/fd6b9aa4f0a843adabd64246b7588d04 2024-11-15T14:15:06,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/dc92041cfbfe4d058759c73642cafa5e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/bd1e0cbe249b44779d2d1eb02fd8f9fd, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/2403f23cfd8c42b386dc853e5cd33c2e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d3c3ba32615041d8be12d6ef1424091e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d1107b37e6d3442eac1f8370ade62825, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/848a5d91bd504086b920192b0744202b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d9325afb22fc4893925cc7e91f709f6b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d512c078f843489a940e7cb5f725854d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/eb7ca1834544454091b8fa84805bcca6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/572064a882744c878d64acaa1ba41492, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/c6e8a6fae42c45b3bfbb4bf26cfe0b11, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d460826e8ae044e9ac76c4e2f972161b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d9589d28c0f1491a92e3c0a2599329c1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/0dfbc9fa356848958cb12b4d9112a252, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/3e2c09ad12434c91879f4293e0f89b36, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/9274949692c8424894fc59024a4f01b3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/8b7afb2cce8c4bf181bc8b1c81f85904, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/e7bafdccee2240cd893151bd139d0301, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fac5980c864440759781fb4675c33763, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/69a0361ebf5a4ae18c30bda1ba89cdf6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/b12f3b54ed80410ebaf701c71d7c9ebc, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/78c4a812bae6484fb1088ed540fe4e88, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/5942a7bba433438ebc9bbce975680f01, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/9a461c71da274168a30c567ed720844b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/a798e9c8ad87458f8caa23bfc87fc6fe, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/351016014e8e4dc9a010a0bca532dab6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/e7a47fcb87ad41fe8dc69991c08513eb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/3daf6276842e40a48e4754be5aab7912, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/0b5eb81700b0496084d45d4a65521af8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/33a55e6b679a4a3f867c9e0a535432c3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/f439c93b0bb94431a4091a9c529981f9] to archive 2024-11-15T14:15:06,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:15:06,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/dc92041cfbfe4d058759c73642cafa5e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/dc92041cfbfe4d058759c73642cafa5e 2024-11-15T14:15:06,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/bd1e0cbe249b44779d2d1eb02fd8f9fd to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/bd1e0cbe249b44779d2d1eb02fd8f9fd 2024-11-15T14:15:06,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/2403f23cfd8c42b386dc853e5cd33c2e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/2403f23cfd8c42b386dc853e5cd33c2e 2024-11-15T14:15:06,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d3c3ba32615041d8be12d6ef1424091e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d3c3ba32615041d8be12d6ef1424091e 2024-11-15T14:15:06,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d1107b37e6d3442eac1f8370ade62825 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d1107b37e6d3442eac1f8370ade62825 2024-11-15T14:15:06,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/848a5d91bd504086b920192b0744202b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/848a5d91bd504086b920192b0744202b 2024-11-15T14:15:06,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d9325afb22fc4893925cc7e91f709f6b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d9325afb22fc4893925cc7e91f709f6b 2024-11-15T14:15:06,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d512c078f843489a940e7cb5f725854d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d512c078f843489a940e7cb5f725854d 2024-11-15T14:15:06,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/eb7ca1834544454091b8fa84805bcca6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/eb7ca1834544454091b8fa84805bcca6 2024-11-15T14:15:06,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/572064a882744c878d64acaa1ba41492 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/572064a882744c878d64acaa1ba41492 2024-11-15T14:15:06,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/c6e8a6fae42c45b3bfbb4bf26cfe0b11 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/c6e8a6fae42c45b3bfbb4bf26cfe0b11 2024-11-15T14:15:06,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d460826e8ae044e9ac76c4e2f972161b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d460826e8ae044e9ac76c4e2f972161b 2024-11-15T14:15:06,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d9589d28c0f1491a92e3c0a2599329c1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/d9589d28c0f1491a92e3c0a2599329c1 2024-11-15T14:15:06,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/0dfbc9fa356848958cb12b4d9112a252 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/0dfbc9fa356848958cb12b4d9112a252 2024-11-15T14:15:06,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/3e2c09ad12434c91879f4293e0f89b36 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/3e2c09ad12434c91879f4293e0f89b36 2024-11-15T14:15:06,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/9274949692c8424894fc59024a4f01b3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/9274949692c8424894fc59024a4f01b3 2024-11-15T14:15:06,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/8b7afb2cce8c4bf181bc8b1c81f85904 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/8b7afb2cce8c4bf181bc8b1c81f85904 2024-11-15T14:15:06,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/e7bafdccee2240cd893151bd139d0301 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/e7bafdccee2240cd893151bd139d0301 2024-11-15T14:15:06,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fac5980c864440759781fb4675c33763 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fac5980c864440759781fb4675c33763 2024-11-15T14:15:06,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/69a0361ebf5a4ae18c30bda1ba89cdf6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/69a0361ebf5a4ae18c30bda1ba89cdf6 2024-11-15T14:15:06,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/b12f3b54ed80410ebaf701c71d7c9ebc to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/b12f3b54ed80410ebaf701c71d7c9ebc 2024-11-15T14:15:06,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/78c4a812bae6484fb1088ed540fe4e88 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/78c4a812bae6484fb1088ed540fe4e88 2024-11-15T14:15:06,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/5942a7bba433438ebc9bbce975680f01 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/5942a7bba433438ebc9bbce975680f01 2024-11-15T14:15:06,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/9a461c71da274168a30c567ed720844b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/9a461c71da274168a30c567ed720844b 2024-11-15T14:15:06,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/a798e9c8ad87458f8caa23bfc87fc6fe to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/a798e9c8ad87458f8caa23bfc87fc6fe 2024-11-15T14:15:06,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/351016014e8e4dc9a010a0bca532dab6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/351016014e8e4dc9a010a0bca532dab6 2024-11-15T14:15:06,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/e7a47fcb87ad41fe8dc69991c08513eb to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/e7a47fcb87ad41fe8dc69991c08513eb 2024-11-15T14:15:06,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/3daf6276842e40a48e4754be5aab7912 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/3daf6276842e40a48e4754be5aab7912 2024-11-15T14:15:06,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/0b5eb81700b0496084d45d4a65521af8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/0b5eb81700b0496084d45d4a65521af8 2024-11-15T14:15:06,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/33a55e6b679a4a3f867c9e0a535432c3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/33a55e6b679a4a3f867c9e0a535432c3 2024-11-15T14:15:06,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/f439c93b0bb94431a4091a9c529981f9 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/f439c93b0bb94431a4091a9c529981f9 2024-11-15T14:15:06,375 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/recovered.edits/506.seqid, newMaxSeqId=506, maxSeqId=1 2024-11-15T14:15:06,378 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. 2024-11-15T14:15:06,378 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 785aa89faccd90f1bafb73964185be68: Waiting for close lock at 1731680104627Running coprocessor pre-close hooks at 1731680104627Disabling compacts and flushes for region at 1731680104627Disabling writes for close at 1731680104963 (+336 ms)Obtaining lock to block concurrent updates at 1731680104963Preparing flush snapshotting stores in 785aa89faccd90f1bafb73964185be68 at 1731680104963Finished memstore snapshotting TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731680104964 (+1 ms)Flushing stores of TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68. at 1731680104964Flushing 785aa89faccd90f1bafb73964185be68/A: creating writer at 1731680104964Flushing 785aa89faccd90f1bafb73964185be68/A: appending metadata at 1731680104967 (+3 ms)Flushing 785aa89faccd90f1bafb73964185be68/A: closing flushed file at 1731680104967Flushing 785aa89faccd90f1bafb73964185be68/B: creating writer at 1731680105380 (+413 ms)Flushing 785aa89faccd90f1bafb73964185be68/B: appending metadata at 1731680105384 (+4 ms)Flushing 785aa89faccd90f1bafb73964185be68/B: closing flushed file at 1731680105384Flushing 785aa89faccd90f1bafb73964185be68/C: creating writer at 1731680105801 (+417 ms)Flushing 785aa89faccd90f1bafb73964185be68/C: appending metadata at 1731680105805 (+4 ms)Flushing 785aa89faccd90f1bafb73964185be68/C: closing flushed file at 1731680105805Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f17f885: reopening flushed file at 1731680106217 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c5c5b5e: reopening flushed file at 1731680106224 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d936049: reopening flushed file at 1731680106229 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 785aa89faccd90f1bafb73964185be68 in 1272ms, sequenceid=503, compaction requested=true at 1731680106235 (+6 ms)Writing region close event to WAL at 1731680106371 (+136 ms)Running coprocessor post-close hooks at 1731680106376 (+5 ms)Closed at 1731680106378 (+2 ms) 2024-11-15T14:15:06,381 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 785aa89faccd90f1bafb73964185be68 2024-11-15T14:15:06,381 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=785aa89faccd90f1bafb73964185be68, regionState=CLOSED 2024-11-15T14:15:06,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:15:06,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-15T14:15:06,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 785aa89faccd90f1bafb73964185be68, server=26754e923c4a,40177,1731680077530 in 1.9190 sec 2024-11-15T14:15:06,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-15T14:15:06,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=785aa89faccd90f1bafb73964185be68, UNASSIGN in 1.9270 sec 2024-11-15T14:15:06,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-15T14:15:06,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9340 sec 2024-11-15T14:15:06,392 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680106391"}]},"ts":"1731680106391"} 2024-11-15T14:15:06,393 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T14:15:06,393 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T14:15:06,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9490 sec 2024-11-15T14:15:06,558 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T14:15:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T14:15:06,602 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:06,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T14:15:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:06,614 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-15T14:15:06,616 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:06,622 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68 2024-11-15T14:15:06,626 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/recovered.edits] 2024-11-15T14:15:06,637 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/15a69e66c6d14dec8008a4302120f558 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/15a69e66c6d14dec8008a4302120f558 2024-11-15T14:15:06,639 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/1e8ec721f80c42858dd4c32d2d7be415 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/1e8ec721f80c42858dd4c32d2d7be415 2024-11-15T14:15:06,640 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/98a27ea4d63546c79545bcff07efb159 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/98a27ea4d63546c79545bcff07efb159 2024-11-15T14:15:06,642 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/bf779987082b495dab9fc85cc66818f2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/A/bf779987082b495dab9fc85cc66818f2 2024-11-15T14:15:06,644 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/5f39a592e1d0406984bf28d160770bf0 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/5f39a592e1d0406984bf28d160770bf0 2024-11-15T14:15:06,645 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/a09c488045c04c5e8c0e096e26532763 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/a09c488045c04c5e8c0e096e26532763 2024-11-15T14:15:06,647 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d71854e68a2b44a68b6af3f99e5e10d1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/d71854e68a2b44a68b6af3f99e5e10d1 2024-11-15T14:15:06,648 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/f09ca7de91d848eeb2c7ca7dbb5a6cb7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/B/f09ca7de91d848eeb2c7ca7dbb5a6cb7 2024-11-15T14:15:06,650 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/7207c3acd24147bbbc4c13bd85dac148 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/7207c3acd24147bbbc4c13bd85dac148 2024-11-15T14:15:06,651 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/79c5b43a663a4dd7805bab06016fcf12 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/79c5b43a663a4dd7805bab06016fcf12 2024-11-15T14:15:06,652 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fa131a91b1764b538bd98b9cc86d07e3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fa131a91b1764b538bd98b9cc86d07e3 2024-11-15T14:15:06,653 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fdf3bcceb55c4e7088e85c78518507f4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/C/fdf3bcceb55c4e7088e85c78518507f4 2024-11-15T14:15:06,656 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/recovered.edits/506.seqid to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68/recovered.edits/506.seqid 2024-11-15T14:15:06,656 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/785aa89faccd90f1bafb73964185be68 2024-11-15T14:15:06,656 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T14:15:06,661 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:06,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-15T14:15:06,670 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T14:15:06,696 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T14:15:06,698 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:06,698 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T14:15:06,698 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731680106698"}]},"ts":"9223372036854775807"} 2024-11-15T14:15:06,702 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T14:15:06,702 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 785aa89faccd90f1bafb73964185be68, NAME => 'TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T14:15:06,702 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T14:15:06,702 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731680106702"}]},"ts":"9223372036854775807"} 2024-11-15T14:15:06,705 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T14:15:06,706 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:06,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 98 msec 2024-11-15T14:15:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-15T14:15:06,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T14:15:06,723 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:06,737 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/26754e923c4a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;26754e923c4a:40177-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1873743887_22 at /127.0.0.1:44146 [Waiting for operation #329] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1762377796_22 at /127.0.0.1:37588 [Waiting for operation #351] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=655 (was 620) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=268 (was 524) 2024-11-15T14:15:06,747 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=655, ProcessCount=11, AvailableMemoryMB=265 2024-11-15T14:15:06,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T14:15:06,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T14:15:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:06,751 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T14:15:06,751 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:06,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-15T14:15:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-15T14:15:06,752 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T14:15:06,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741944_1120 (size=1061) 2024-11-15T14:15:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-15T14:15:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-15T14:15:07,161 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931 2024-11-15T14:15:07,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741945_1121 (size=53) 2024-11-15T14:15:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-15T14:15:07,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:15:07,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1fd20151ae970369f5abd5f0c2e6e70d, disabling compactions & flushes 2024-11-15T14:15:07,602 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:07,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:07,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. after waiting 0 ms 2024-11-15T14:15:07,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:07,602 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:07,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1fd20151ae970369f5abd5f0c2e6e70d: Waiting for close lock at 1731680107602Disabling compacts and flushes for region at 1731680107602Disabling writes for close at 1731680107602Writing region close event to WAL at 1731680107602Closed at 1731680107602 2024-11-15T14:15:07,604 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T14:15:07,604 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731680107604"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731680107604"}]},"ts":"1731680107604"} 2024-11-15T14:15:07,606 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T14:15:07,608 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T14:15:07,609 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680107608"}]},"ts":"1731680107608"} 2024-11-15T14:15:07,611 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T14:15:07,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, ASSIGN}] 2024-11-15T14:15:07,613 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, ASSIGN 2024-11-15T14:15:07,614 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, ASSIGN; state=OFFLINE, location=26754e923c4a,40177,1731680077530; forceNewPlan=false, retain=false 2024-11-15T14:15:07,764 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=1fd20151ae970369f5abd5f0c2e6e70d, regionState=OPENING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:15:07,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, ASSIGN because future has completed 2024-11-15T14:15:07,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:15:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-15T14:15:07,925 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:07,926 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} 2024-11-15T14:15:07,926 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:07,926 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:15:07,926 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:07,926 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:07,928 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:07,930 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:15:07,930 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fd20151ae970369f5abd5f0c2e6e70d columnFamilyName A 2024-11-15T14:15:07,931 DEBUG [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:07,932 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.HStore(327): Store=1fd20151ae970369f5abd5f0c2e6e70d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:15:07,932 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:07,936 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:15:07,937 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fd20151ae970369f5abd5f0c2e6e70d columnFamilyName B 2024-11-15T14:15:07,937 DEBUG [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:07,939 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.HStore(327): Store=1fd20151ae970369f5abd5f0c2e6e70d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:15:07,939 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:07,941 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:15:07,941 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fd20151ae970369f5abd5f0c2e6e70d columnFamilyName C 2024-11-15T14:15:07,941 DEBUG [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:07,942 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.HStore(327): Store=1fd20151ae970369f5abd5f0c2e6e70d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:15:07,943 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:07,943 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:07,944 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:07,944 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:07,946 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:07,946 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:07,947 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T14:15:07,948 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:07,953 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T14:15:07,954 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 1fd20151ae970369f5abd5f0c2e6e70d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67095364, jitterRate=-2.0116567611694336E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T14:15:07,955 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:07,956 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 1fd20151ae970369f5abd5f0c2e6e70d: Running coprocessor pre-open hook at 1731680107926Writing region info on filesystem at 1731680107926Initializing all the Stores at 1731680107927 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680107927Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680107927Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680107927Cleaning up temporary data from old regions at 1731680107946 (+19 ms)Running coprocessor post-open hooks at 1731680107955 (+9 ms)Region opened successfully at 1731680107956 (+1 ms) 2024-11-15T14:15:07,958 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., pid=32, masterSystemTime=1731680107921 2024-11-15T14:15:07,960 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:07,961 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:07,962 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=1fd20151ae970369f5abd5f0c2e6e70d, regionState=OPEN, openSeqNum=2, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:15:07,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:15:07,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-15T14:15:07,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 in 198 msec 2024-11-15T14:15:07,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-15T14:15:07,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, ASSIGN in 359 msec 2024-11-15T14:15:07,973 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T14:15:07,973 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680107973"}]},"ts":"1731680107973"} 2024-11-15T14:15:07,975 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T14:15:07,977 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T14:15:07,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2280 sec 2024-11-15T14:15:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-15T14:15:08,902 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:08,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:08,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:08,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:08,908 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7287a320, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:08,911 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:08,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2766031b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:08,913 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:08,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:08,916 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:08,920 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T14:15:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 26754e923c4a,35807,1731680075796 2024-11-15T14:15:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b72cf43 2024-11-15T14:15:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T14:15:08,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T14:15:08,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T14:15:08,933 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T14:15:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:08,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741946_1122 (size=1097) 2024-11-15T14:15:08,984 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T14:15:08,984 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T14:15:08,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T14:15:09,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, REOPEN/MOVE}] 2024-11-15T14:15:09,001 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, REOPEN/MOVE 2024-11-15T14:15:09,003 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=1fd20151ae970369f5abd5f0c2e6e70d, regionState=CLOSING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:15:09,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, REOPEN/MOVE because future has completed 2024-11-15T14:15:09,007 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T14:15:09,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:15:09,161 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,161 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T14:15:09,162 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 1fd20151ae970369f5abd5f0c2e6e70d, disabling compactions & flushes 2024-11-15T14:15:09,162 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:09,162 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:09,162 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. after waiting 0 ms 2024-11-15T14:15:09,162 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:09,169 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T14:15:09,170 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:09,170 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 1fd20151ae970369f5abd5f0c2e6e70d: Waiting for close lock at 1731680109162Running coprocessor pre-close hooks at 1731680109162Disabling compacts and flushes for region at 1731680109162Disabling writes for close at 1731680109162Writing region close event to WAL at 1731680109162Running coprocessor post-close hooks at 1731680109170 (+8 ms)Closed at 1731680109170 2024-11-15T14:15:09,171 WARN [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 1fd20151ae970369f5abd5f0c2e6e70d to self. 2024-11-15T14:15:09,173 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,174 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=1fd20151ae970369f5abd5f0c2e6e70d, regionState=CLOSED 2024-11-15T14:15:09,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:15:09,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-15T14:15:09,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 in 170 msec 2024-11-15T14:15:09,180 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, REOPEN/MOVE; state=CLOSED, location=26754e923c4a,40177,1731680077530; forceNewPlan=false, retain=true 2024-11-15T14:15:09,331 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=1fd20151ae970369f5abd5f0c2e6e70d, regionState=OPENING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:15:09,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, REOPEN/MOVE because future has completed 2024-11-15T14:15:09,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:15:09,488 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:09,488 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} 2024-11-15T14:15:09,488 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,489 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:15:09,489 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,489 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,492 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,492 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:15:09,497 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fd20151ae970369f5abd5f0c2e6e70d columnFamilyName A 2024-11-15T14:15:09,500 DEBUG [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:09,501 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.HStore(327): Store=1fd20151ae970369f5abd5f0c2e6e70d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:15:09,502 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,503 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:15:09,503 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fd20151ae970369f5abd5f0c2e6e70d columnFamilyName B 2024-11-15T14:15:09,503 DEBUG [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:09,503 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.HStore(327): Store=1fd20151ae970369f5abd5f0c2e6e70d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:15:09,504 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,504 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:15:09,505 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fd20151ae970369f5abd5f0c2e6e70d columnFamilyName C 2024-11-15T14:15:09,505 DEBUG [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:09,505 INFO [StoreOpener-1fd20151ae970369f5abd5f0c2e6e70d-1 {}] regionserver.HStore(327): Store=1fd20151ae970369f5abd5f0c2e6e70d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:15:09,505 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:09,505 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,506 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,507 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,508 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,508 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,508 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T14:15:09,510 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,511 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 1fd20151ae970369f5abd5f0c2e6e70d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68951325, jitterRate=0.027454808354377747}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T14:15:09,511 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,512 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 1fd20151ae970369f5abd5f0c2e6e70d: Running coprocessor pre-open hook at 1731680109489Writing region info on filesystem at 1731680109489Initializing all the Stores at 1731680109490 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680109490Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680109490Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680109490Cleaning up temporary data from old regions at 1731680109508 (+18 ms)Running coprocessor post-open hooks at 1731680109511 (+3 ms)Region opened successfully at 1731680109512 (+1 ms) 2024-11-15T14:15:09,513 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., pid=37, masterSystemTime=1731680109485 2024-11-15T14:15:09,515 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:09,515 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:09,516 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=1fd20151ae970369f5abd5f0c2e6e70d, regionState=OPEN, openSeqNum=5, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:15:09,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:15:09,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-15T14:15:09,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 in 184 msec 2024-11-15T14:15:09,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-15T14:15:09,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, REOPEN/MOVE in 524 msec 2024-11-15T14:15:09,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-15T14:15:09,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 540 msec 2024-11-15T14:15:09,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 595 msec 2024-11-15T14:15:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-15T14:15:09,537 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:09,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:09,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:09,540 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:09,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f4ad13e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:09,542 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1745a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:09,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:09,545 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dbb8645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:09,547 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7024c837, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:09,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:09,550 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3111d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:09,552 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:09,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62903dc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:09,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:09,556 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f8e1332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:09,558 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f31037b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,559 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:09,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:09,561 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7065d1ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:09,562 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b78af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:09,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:09,565 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@628fd69d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:09,568 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a9dc9dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:09,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:09,574 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e8d81e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:09,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:09,575 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:09,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3034141e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:09,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:09,578 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4938e2ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:09,579 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:09,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a94543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:09,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:09,581 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39349845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:09,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:09,584 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8288ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:09,585 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:09,585 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:09,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:09,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:09,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:09,591 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:09,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:15:09,598 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:09,598 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:15:09,599 DEBUG [Thread-629 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:09,600 DEBUG [Thread-629 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:09,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:09,601 DEBUG [Thread-631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:15:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:09,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:09,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:15:09,611 DEBUG [Thread-618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:09,611 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:09,612 DEBUG [Thread-620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-15T14:15:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:15:09,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T14:15:09,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:09,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:09,616 DEBUG [Thread-620 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:09,617 DEBUG [Thread-620 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:09,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:09,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:15:09,620 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:09,621 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:09,623 DEBUG [Thread-633 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:09,623 DEBUG [Thread-633 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:09,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:15:09,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:15:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T14:15:09,626 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:09,626 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:09,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:09,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:09,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:15:09,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:09,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:09,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:09,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:09,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:09,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b2437f0e6335459ca616c86c34aba507_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680109625/Put/seqid=0 2024-11-15T14:15:09,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680169684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:09,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731680109652, max timestamp=1731680109697Num uniques 859; 2024-11-15T14:15:09,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680169689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:09,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731680109652, max timestamp=1731680109697Num uniques 859; 2024-11-15T14:15:09,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-15T14:15:09,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:09,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731680109652, max timestamp=1731680109697Num uniques 859; 2024-11-15T14:15:09,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-15T14:15:09,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-15T14:15:09,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741947_1123 (size=9714) 2024-11-15T14:15:09,717 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T14:15:09,724 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b2437f0e6335459ca616c86c34aba507_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b2437f0e6335459ca616c86c34aba507_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:09,726 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/ee812d33acde4d6dab7bc447c2fa9b73, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:09,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731680169749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:09,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:09,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731680169755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:09,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:09,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/ee812d33acde4d6dab7bc447c2fa9b73 is 175, key is test_row_0/A:col10/1731680109625/Put/seqid=0 2024-11-15T14:15:09,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33914 deadline: 1731680169757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:09,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T14:15:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:09,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:09,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741948_1124 (size=22361) 2024-11-15T14:15:09,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680169802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:09,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680169812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:09,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731680169862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:09,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:09,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731680169872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:09,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33914 deadline: 1731680169882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:09,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:09,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T14:15:09,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T14:15:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:09,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680170022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680170032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731680170083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T14:15:10,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:10,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:10,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:10,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731680170092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33914 deadline: 1731680170092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,209 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/ee812d33acde4d6dab7bc447c2fa9b73 2024-11-15T14:15:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T14:15:10,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T14:15:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:10,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:10,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:10,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/1b4a69612326482faec3e1003fdcdbfa is 50, key is test_row_0/B:col10/1731680109625/Put/seqid=0 2024-11-15T14:15:10,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741949_1125 (size=9657) 2024-11-15T14:15:10,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/1b4a69612326482faec3e1003fdcdbfa 2024-11-15T14:15:10,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/cc2105cc41474ab1b4305245e8e0dc5b is 50, key is test_row_0/C:col10/1731680109625/Put/seqid=0 2024-11-15T14:15:10,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680170343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680170343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741950_1126 (size=9657) 2024-11-15T14:15:10,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T14:15:10,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:10,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:10,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:10,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731680170402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731680170412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33914 deadline: 1731680170412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T14:15:10,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:10,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:10,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:10,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T14:15:10,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:10,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:10,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:10,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T14:15:10,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/cc2105cc41474ab1b4305245e8e0dc5b 2024-11-15T14:15:10,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/ee812d33acde4d6dab7bc447c2fa9b73 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee812d33acde4d6dab7bc447c2fa9b73 2024-11-15T14:15:10,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee812d33acde4d6dab7bc447c2fa9b73, entries=100, sequenceid=15, filesize=21.8 K 2024-11-15T14:15:10,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/1b4a69612326482faec3e1003fdcdbfa as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/1b4a69612326482faec3e1003fdcdbfa 2024-11-15T14:15:10,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/1b4a69612326482faec3e1003fdcdbfa, entries=100, sequenceid=15, filesize=9.4 K 2024-11-15T14:15:10,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/cc2105cc41474ab1b4305245e8e0dc5b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/cc2105cc41474ab1b4305245e8e0dc5b 2024-11-15T14:15:10,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/cc2105cc41474ab1b4305245e8e0dc5b, entries=100, sequenceid=15, filesize=9.4 K 2024-11-15T14:15:10,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 1fd20151ae970369f5abd5f0c2e6e70d in 1219ms, sequenceid=15, compaction requested=false 2024-11-15T14:15:10,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:10,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T14:15:10,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:10,869 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=134.18 KB heapSize=361.55 KB 2024-11-15T14:15:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:10,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:10,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:10,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:10,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:10,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:10,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:10,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115faae6fbe1d2d443e950f4267ac607628_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680109713/Put/seqid=0 2024-11-15T14:15:10,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680170907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731680170923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680170924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741951_1127 (size=14594) 2024-11-15T14:15:10,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731680170933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33914 deadline: 1731680170933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:10,951 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115faae6fbe1d2d443e950f4267ac607628_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115faae6fbe1d2d443e950f4267ac607628_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:10,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/ee1d53373c4e4a7aaa49f2b09346ce9c, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:10,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/ee1d53373c4e4a7aaa49f2b09346ce9c is 175, key is test_row_0/A:col10/1731680109713/Put/seqid=0 2024-11-15T14:15:10,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741952_1128 (size=39549) 2024-11-15T14:15:10,988 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/ee1d53373c4e4a7aaa49f2b09346ce9c 2024-11-15T14:15:11,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/ce987213503b482298811f328e488bc1 is 50, key is test_row_0/B:col10/1731680109713/Put/seqid=0 2024-11-15T14:15:11,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680171042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:11,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680171052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:11,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741953_1129 (size=12001) 2024-11-15T14:15:11,067 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/ce987213503b482298811f328e488bc1 2024-11-15T14:15:11,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/0794f4b2956a4bb0995447ba470cf361 is 50, key is test_row_0/C:col10/1731680109713/Put/seqid=0 2024-11-15T14:15:11,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741954_1130 (size=12001) 2024-11-15T14:15:11,146 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/0794f4b2956a4bb0995447ba470cf361 2024-11-15T14:15:11,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/ee1d53373c4e4a7aaa49f2b09346ce9c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee1d53373c4e4a7aaa49f2b09346ce9c 2024-11-15T14:15:11,162 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee1d53373c4e4a7aaa49f2b09346ce9c, entries=200, sequenceid=48, filesize=38.6 K 2024-11-15T14:15:11,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/ce987213503b482298811f328e488bc1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/ce987213503b482298811f328e488bc1 2024-11-15T14:15:11,187 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/ce987213503b482298811f328e488bc1, entries=150, sequenceid=48, filesize=11.7 K 2024-11-15T14:15:11,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/0794f4b2956a4bb0995447ba470cf361 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/0794f4b2956a4bb0995447ba470cf361 2024-11-15T14:15:11,206 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/0794f4b2956a4bb0995447ba470cf361, entries=150, sequenceid=48, filesize=11.7 K 2024-11-15T14:15:11,207 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=46.96 KB/48090 for 1fd20151ae970369f5abd5f0c2e6e70d in 339ms, sequenceid=48, compaction requested=false 2024-11-15T14:15:11,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:11,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:11,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-15T14:15:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-15T14:15:11,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-15T14:15:11,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5830 sec 2024-11-15T14:15:11,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.6160 sec 2024-11-15T14:15:11,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T14:15:11,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:11,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:11,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:11,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:11,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:11,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:11,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111584b4f0d625d84e20a48038f7a0f80ebe_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680110905/Put/seqid=0 2024-11-15T14:15:11,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741955_1131 (size=14594) 2024-11-15T14:15:11,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680111301, max timestamp=1731680111377Num uniques 859; 2024-11-15T14:15:11,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680111301, max timestamp=1731680111377Num uniques 859; 2024-11-15T14:15:11,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:11,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680111301, max timestamp=1731680111377Num uniques 859; 2024-11-15T14:15:11,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:11,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:11,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680171512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:11,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:11,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680171529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:11,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:11,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680171632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:11,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:11,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680171642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:11,752 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:11,761 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111584b4f0d625d84e20a48038f7a0f80ebe_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111584b4f0d625d84e20a48038f7a0f80ebe_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T14:15:11,762 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/18acfa812f1642598a5f669be1d27f03, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:11,763 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:11,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/18acfa812f1642598a5f669be1d27f03 is 175, key is test_row_0/A:col10/1731680110905/Put/seqid=0 2024-11-15T14:15:11,764 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-15T14:15:11,767 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:11,768 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:11,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T14:15:11,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741956_1132 (size=39549) 2024-11-15T14:15:11,799 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/18acfa812f1642598a5f669be1d27f03 2024-11-15T14:15:11,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/9a150f956d9e4b3cbfef772573b19ebf is 50, key is test_row_0/B:col10/1731680110905/Put/seqid=0 2024-11-15T14:15:11,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680171842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:11,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680171852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:11,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:11,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741957_1133 (size=12001) 2024-11-15T14:15:11,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/9a150f956d9e4b3cbfef772573b19ebf 2024-11-15T14:15:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T14:15:11,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/53d39863c3b241a5bdec6f5a377312b6 is 50, key is test_row_0/C:col10/1731680110905/Put/seqid=0 2024-11-15T14:15:11,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T14:15:11,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:11,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:11,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:11,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:11,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:11,946 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T14:15:11,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741958_1134 (size=12001) 2024-11-15T14:15:11,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/53d39863c3b241a5bdec6f5a377312b6 2024-11-15T14:15:11,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/18acfa812f1642598a5f669be1d27f03 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/18acfa812f1642598a5f669be1d27f03 2024-11-15T14:15:11,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/18acfa812f1642598a5f669be1d27f03, entries=200, sequenceid=59, filesize=38.6 K 2024-11-15T14:15:11,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/9a150f956d9e4b3cbfef772573b19ebf as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9a150f956d9e4b3cbfef772573b19ebf 2024-11-15T14:15:11,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731680171951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:11,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33914 deadline: 1731680171951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:11,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731680171963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:11,984 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in 1fd20151ae970369f5abd5f0c2e6e70d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:11,984 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in 1fd20151ae970369f5abd5f0c2e6e70d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:11,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:11,984 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in 1fd20151ae970369f5abd5f0c2e6e70d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:11,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9a150f956d9e4b3cbfef772573b19ebf, entries=150, sequenceid=59, filesize=11.7 K 2024-11-15T14:15:11,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/53d39863c3b241a5bdec6f5a377312b6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/53d39863c3b241a5bdec6f5a377312b6 2024-11-15T14:15:11,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/53d39863c3b241a5bdec6f5a377312b6, entries=150, sequenceid=59, filesize=11.7 K 2024-11-15T14:15:11,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1fd20151ae970369f5abd5f0c2e6e70d in 740ms, sequenceid=59, compaction requested=true 2024-11-15T14:15:11,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:11,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:11,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:11,999 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:11,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:11,999 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:11,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:11,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:11,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:12,000 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:12,000 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:12,000 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/B is initiating minor compaction (all files) 2024-11-15T14:15:12,000 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/A is initiating minor compaction (all files) 2024-11-15T14:15:12,000 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/B in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:12,000 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/A in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:12,000 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/1b4a69612326482faec3e1003fdcdbfa, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/ce987213503b482298811f328e488bc1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9a150f956d9e4b3cbfef772573b19ebf] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=32.9 K 2024-11-15T14:15:12,000 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee812d33acde4d6dab7bc447c2fa9b73, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee1d53373c4e4a7aaa49f2b09346ce9c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/18acfa812f1642598a5f669be1d27f03] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=99.1 K 2024-11-15T14:15:12,001 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:12,001 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b4a69612326482faec3e1003fdcdbfa, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731680109621 2024-11-15T14:15:12,001 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee812d33acde4d6dab7bc447c2fa9b73, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee1d53373c4e4a7aaa49f2b09346ce9c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/18acfa812f1642598a5f669be1d27f03] 2024-11-15T14:15:12,002 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting ce987213503b482298811f328e488bc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731680109713 2024-11-15T14:15:12,003 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee812d33acde4d6dab7bc447c2fa9b73, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731680109621 2024-11-15T14:15:12,003 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a150f956d9e4b3cbfef772573b19ebf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731680110885 2024-11-15T14:15:12,006 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee1d53373c4e4a7aaa49f2b09346ce9c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731680109713 2024-11-15T14:15:12,006 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18acfa812f1642598a5f669be1d27f03, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731680110885 2024-11-15T14:15:12,019 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:12,020 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#B#compaction#116 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:12,021 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/9babfcf492474a5db90f2efe43906740 is 50, key is test_row_0/B:col10/1731680110905/Put/seqid=0 2024-11-15T14:15:12,048 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115881ca647579541a88b5f856aa77652c2_1fd20151ae970369f5abd5f0c2e6e70d store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:12,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741959_1135 (size=12104) 2024-11-15T14:15:12,053 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115881ca647579541a88b5f856aa77652c2_1fd20151ae970369f5abd5f0c2e6e70d, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:12,054 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115881ca647579541a88b5f856aa77652c2_1fd20151ae970369f5abd5f0c2e6e70d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:12,063 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/9babfcf492474a5db90f2efe43906740 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9babfcf492474a5db90f2efe43906740 2024-11-15T14:15:12,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T14:15:12,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:12,076 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-15T14:15:12,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:12,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:12,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T14:15:12,084 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/B of 1fd20151ae970369f5abd5f0c2e6e70d into 9babfcf492474a5db90f2efe43906740(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:12,084 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:12,084 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/B, priority=13, startTime=1731680111999; duration=0sec 2024-11-15T14:15:12,085 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:12,085 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:B 2024-11-15T14:15:12,085 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:12,087 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:12,087 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/C is initiating minor compaction (all files) 2024-11-15T14:15:12,087 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/C in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:12,088 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/cc2105cc41474ab1b4305245e8e0dc5b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/0794f4b2956a4bb0995447ba470cf361, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/53d39863c3b241a5bdec6f5a377312b6] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=32.9 K 2024-11-15T14:15:12,089 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting cc2105cc41474ab1b4305245e8e0dc5b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731680109621 2024-11-15T14:15:12,089 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0794f4b2956a4bb0995447ba470cf361, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731680109713 2024-11-15T14:15:12,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741960_1136 (size=4469) 2024-11-15T14:15:12,090 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d39863c3b241a5bdec6f5a377312b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731680110885 2024-11-15T14:15:12,104 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#A#compaction#117 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:12,104 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/5bcf2a2ca5f140998e10f7184f487c17 is 175, key is test_row_0/A:col10/1731680110905/Put/seqid=0 2024-11-15T14:15:12,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150fb79dc5987944e7805c834efdd10758_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680111490/Put/seqid=0 2024-11-15T14:15:12,142 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#C#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:12,142 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/106a6752f6224de2b0dbcfe9e66aa157 is 50, key is test_row_0/C:col10/1731680110905/Put/seqid=0 2024-11-15T14:15:12,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741961_1137 (size=31058) 2024-11-15T14:15:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:12,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:12,168 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/5bcf2a2ca5f140998e10f7184f487c17 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/5bcf2a2ca5f140998e10f7184f487c17 2024-11-15T14:15:12,177 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/A of 1fd20151ae970369f5abd5f0c2e6e70d into 5bcf2a2ca5f140998e10f7184f487c17(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:12,177 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:12,177 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/A, priority=13, startTime=1731680111999; duration=0sec 2024-11-15T14:15:12,177 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:12,178 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:A 2024-11-15T14:15:12,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680172192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:12,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741963_1139 (size=12104) 2024-11-15T14:15:12,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741962_1138 (size=12154) 2024-11-15T14:15:12,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:12,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680172199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:12,218 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/106a6752f6224de2b0dbcfe9e66aa157 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/106a6752f6224de2b0dbcfe9e66aa157 2024-11-15T14:15:12,221 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150fb79dc5987944e7805c834efdd10758_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150fb79dc5987944e7805c834efdd10758_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:12,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/7047870c23cd49808bcd8e9044f6106b, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:12,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/7047870c23cd49808bcd8e9044f6106b is 175, key is test_row_0/A:col10/1731680111490/Put/seqid=0 2024-11-15T14:15:12,240 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/C of 1fd20151ae970369f5abd5f0c2e6e70d into 106a6752f6224de2b0dbcfe9e66aa157(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:12,240 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:12,240 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/C, priority=13, startTime=1731680111999; duration=0sec 2024-11-15T14:15:12,241 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:12,241 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:C 2024-11-15T14:15:12,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741964_1140 (size=30955) 2024-11-15T14:15:12,255 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/7047870c23cd49808bcd8e9044f6106b 2024-11-15T14:15:12,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/a55fa84b26f240978c8cb2eaf4c5bcf4 is 50, key is test_row_0/B:col10/1731680111490/Put/seqid=0 2024-11-15T14:15:12,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680172311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:12,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680172323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:12,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:12,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741965_1141 (size=12001) 2024-11-15T14:15:12,338 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/a55fa84b26f240978c8cb2eaf4c5bcf4 2024-11-15T14:15:12,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/dce8cadfd87b460cb8080027746187c2 is 50, key is test_row_0/C:col10/1731680111490/Put/seqid=0 2024-11-15T14:15:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T14:15:12,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741966_1142 (size=12001) 2024-11-15T14:15:12,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680172522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:12,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680172533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:12,832 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/dce8cadfd87b460cb8080027746187c2 2024-11-15T14:15:12,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/7047870c23cd49808bcd8e9044f6106b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7047870c23cd49808bcd8e9044f6106b 2024-11-15T14:15:12,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680172832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:12,845 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7047870c23cd49808bcd8e9044f6106b, entries=150, sequenceid=95, filesize=30.2 K 2024-11-15T14:15:12,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/a55fa84b26f240978c8cb2eaf4c5bcf4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a55fa84b26f240978c8cb2eaf4c5bcf4 2024-11-15T14:15:12,856 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a55fa84b26f240978c8cb2eaf4c5bcf4, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T14:15:12,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680172851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:12,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/dce8cadfd87b460cb8080027746187c2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/dce8cadfd87b460cb8080027746187c2 2024-11-15T14:15:12,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:12,868 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/dce8cadfd87b460cb8080027746187c2, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T14:15:12,870 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 1fd20151ae970369f5abd5f0c2e6e70d in 794ms, sequenceid=95, compaction requested=false 2024-11-15T14:15:12,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:12,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:12,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-15T14:15:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-15T14:15:12,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-15T14:15:12,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1040 sec 2024-11-15T14:15:12,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.1110 sec 2024-11-15T14:15:12,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T14:15:12,903 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:12,904 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-15T14:15:12,907 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T14:15:12,909 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:12,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T14:15:13,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T14:15:13,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:13,063 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T14:15:13,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:13,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:13,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:13,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:13,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:13,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:13,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155f19d055d2e9428aa864155aed2ec6fc_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680112187/Put/seqid=0 2024-11-15T14:15:13,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741967_1143 (size=12154) 2024-11-15T14:15:13,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:13,145 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155f19d055d2e9428aa864155aed2ec6fc_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155f19d055d2e9428aa864155aed2ec6fc_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:13,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/4181c72c2d78450f83cb2a328632c4d9, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:13,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/4181c72c2d78450f83cb2a328632c4d9 is 175, key is test_row_0/A:col10/1731680112187/Put/seqid=0 2024-11-15T14:15:13,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741968_1144 (size=30955) 2024-11-15T14:15:13,190 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/4181c72c2d78450f83cb2a328632c4d9 2024-11-15T14:15:13,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/f239d11ccb154802ae608b5b4b18098b is 50, key is test_row_0/B:col10/1731680112187/Put/seqid=0 2024-11-15T14:15:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T14:15:13,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741969_1145 (size=12001) 2024-11-15T14:15:13,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:13,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680113416, max timestamp=1731680113445Num uniques 859; 2024-11-15T14:15:13,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:15:13,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680113416, max timestamp=1731680113445Num uniques 859; 2024-11-15T14:15:13,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:15:13,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680113416, max timestamp=1731680113445Num uniques 859; 2024-11-15T14:15:13,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:15:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T14:15:13,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680173529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:13,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680173537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:13,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680173642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:13,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680173643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:13,659 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/f239d11ccb154802ae608b5b4b18098b 2024-11-15T14:15:13,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/1fc651ca16a84da19172d6d6dc993389 is 50, key is test_row_0/C:col10/1731680112187/Put/seqid=0 2024-11-15T14:15:13,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741970_1146 (size=12001) 2024-11-15T14:15:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680173862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680173862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:13,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33914 deadline: 1731680173992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:13,994 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in 1fd20151ae970369f5abd5f0c2e6e70d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:14,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731680174003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:14,009 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in 1fd20151ae970369f5abd5f0c2e6e70d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:14,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731680174012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:14,016 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in 1fd20151ae970369f5abd5f0c2e6e70d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:14,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T14:15:14,112 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/1fc651ca16a84da19172d6d6dc993389 2024-11-15T14:15:14,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/4181c72c2d78450f83cb2a328632c4d9 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4181c72c2d78450f83cb2a328632c4d9 2024-11-15T14:15:14,132 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4181c72c2d78450f83cb2a328632c4d9, entries=150, sequenceid=109, filesize=30.2 K 2024-11-15T14:15:14,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/f239d11ccb154802ae608b5b4b18098b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f239d11ccb154802ae608b5b4b18098b 2024-11-15T14:15:14,148 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f239d11ccb154802ae608b5b4b18098b, entries=150, sequenceid=109, filesize=11.7 K 2024-11-15T14:15:14,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/1fc651ca16a84da19172d6d6dc993389 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1fc651ca16a84da19172d6d6dc993389 2024-11-15T14:15:14,163 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1fc651ca16a84da19172d6d6dc993389, entries=150, sequenceid=109, filesize=11.7 K 2024-11-15T14:15:14,165 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1fd20151ae970369f5abd5f0c2e6e70d in 1102ms, sequenceid=109, compaction requested=true 2024-11-15T14:15:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-15T14:15:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-15T14:15:14,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-15T14:15:14,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2580 sec 2024-11-15T14:15:14,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.2680 sec 2024-11-15T14:15:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:14,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-15T14:15:14,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:14,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:14,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154ad96287d3d94ac4a8ee4890dbfa039f_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680113484/Put/seqid=0 2024-11-15T14:15:14,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680174210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:14,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:14,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680174213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:14,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741971_1147 (size=14794) 2024-11-15T14:15:14,234 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:14,241 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154ad96287d3d94ac4a8ee4890dbfa039f_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154ad96287d3d94ac4a8ee4890dbfa039f_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:14,243 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/0b9cdbe1342040e48ca5b03d040472b1, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:14,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/0b9cdbe1342040e48ca5b03d040472b1 is 175, key is test_row_0/A:col10/1731680113484/Put/seqid=0 2024-11-15T14:15:14,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741972_1148 (size=39749) 2024-11-15T14:15:14,263 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/0b9cdbe1342040e48ca5b03d040472b1 2024-11-15T14:15:14,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/c45b9d77ffb14c7b816281f5ecdfd0eb is 50, key is test_row_0/B:col10/1731680113484/Put/seqid=0 2024-11-15T14:15:14,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680174321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:14,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680174332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:14,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741973_1149 (size=12151) 2024-11-15T14:15:14,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680174532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:14,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:14,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680174542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:14,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:14,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/c45b9d77ffb14c7b816281f5ecdfd0eb 2024-11-15T14:15:14,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/42adeb719b0346a4ae36cc365927e12a is 50, key is test_row_0/C:col10/1731680113484/Put/seqid=0 2024-11-15T14:15:14,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741974_1150 (size=12151) 2024-11-15T14:15:14,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680174843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:14,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680174862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:15,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T14:15:15,052 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:15,053 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:15,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-15T14:15:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T14:15:15,055 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:15,056 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:15,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T14:15:15,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T14:15:15,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:15,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:15,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:15,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:15,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:15,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/42adeb719b0346a4ae36cc365927e12a 2024-11-15T14:15:15,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/0b9cdbe1342040e48ca5b03d040472b1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0b9cdbe1342040e48ca5b03d040472b1 2024-11-15T14:15:15,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0b9cdbe1342040e48ca5b03d040472b1, entries=200, sequenceid=145, filesize=38.8 K 2024-11-15T14:15:15,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/c45b9d77ffb14c7b816281f5ecdfd0eb as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/c45b9d77ffb14c7b816281f5ecdfd0eb 2024-11-15T14:15:15,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/c45b9d77ffb14c7b816281f5ecdfd0eb, entries=150, sequenceid=145, filesize=11.9 K 2024-11-15T14:15:15,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/42adeb719b0346a4ae36cc365927e12a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/42adeb719b0346a4ae36cc365927e12a 2024-11-15T14:15:15,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/42adeb719b0346a4ae36cc365927e12a, entries=150, sequenceid=145, filesize=11.9 K 2024-11-15T14:15:15,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 1fd20151ae970369f5abd5f0c2e6e70d in 1104ms, sequenceid=145, compaction requested=true 2024-11-15T14:15:15,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:15,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:15,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:15,284 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:15,284 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:15,287 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132717 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:15,287 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:15,287 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/A is initiating minor compaction (all files) 2024-11-15T14:15:15,287 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/B is initiating minor compaction (all files) 2024-11-15T14:15:15,287 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/B in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:15,287 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/A in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:15,287 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/5bcf2a2ca5f140998e10f7184f487c17, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7047870c23cd49808bcd8e9044f6106b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4181c72c2d78450f83cb2a328632c4d9, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0b9cdbe1342040e48ca5b03d040472b1] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=129.6 K 2024-11-15T14:15:15,287 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9babfcf492474a5db90f2efe43906740, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a55fa84b26f240978c8cb2eaf4c5bcf4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f239d11ccb154802ae608b5b4b18098b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/c45b9d77ffb14c7b816281f5ecdfd0eb] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=47.1 K 2024-11-15T14:15:15,287 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:15,288 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/5bcf2a2ca5f140998e10f7184f487c17, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7047870c23cd49808bcd8e9044f6106b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4181c72c2d78450f83cb2a328632c4d9, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0b9cdbe1342040e48ca5b03d040472b1] 2024-11-15T14:15:15,288 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bcf2a2ca5f140998e10f7184f487c17, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731680110885 2024-11-15T14:15:15,288 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9babfcf492474a5db90f2efe43906740, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731680110885 2024-11-15T14:15:15,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:15,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:15,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:15,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:15,288 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting a55fa84b26f240978c8cb2eaf4c5bcf4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731680111470 2024-11-15T14:15:15,288 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7047870c23cd49808bcd8e9044f6106b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731680111470 2024-11-15T14:15:15,289 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4181c72c2d78450f83cb2a328632c4d9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731680112157 2024-11-15T14:15:15,289 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting f239d11ccb154802ae608b5b4b18098b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731680112157 2024-11-15T14:15:15,289 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b9cdbe1342040e48ca5b03d040472b1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731680113484 2024-11-15T14:15:15,290 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c45b9d77ffb14c7b816281f5ecdfd0eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731680113484 2024-11-15T14:15:15,310 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#B#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:15,311 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/87384dda24c342fabfc8199beff79ecf is 50, key is test_row_0/B:col10/1731680113484/Put/seqid=0 2024-11-15T14:15:15,323 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:15,355 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111541eabca3aac245cab8654cca0fd1b2ff_1fd20151ae970369f5abd5f0c2e6e70d store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:15,360 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111541eabca3aac245cab8654cca0fd1b2ff_1fd20151ae970369f5abd5f0c2e6e70d, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:15,360 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111541eabca3aac245cab8654cca0fd1b2ff_1fd20151ae970369f5abd5f0c2e6e70d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:15,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T14:15:15,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:15,364 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T14:15:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:15,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T14:15:15,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741975_1151 (size=12391) 2024-11-15T14:15:15,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741976_1152 (size=4469) 2024-11-15T14:15:15,421 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#A#compaction#129 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:15,422 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/fc3f178d02094dc0ae728ffb4b79e504 is 175, key is test_row_0/A:col10/1731680113484/Put/seqid=0 2024-11-15T14:15:15,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155490cd06c03f4773b81b2e23ca1222cd_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680114195/Put/seqid=0 2024-11-15T14:15:15,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741977_1153 (size=31345) 2024-11-15T14:15:15,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731680115383, max timestamp=1731680115477Num uniques 859; 2024-11-15T14:15:15,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T14:15:15,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731680115383, max timestamp=1731680115477Num uniques 859; 2024-11-15T14:15:15,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T14:15:15,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741978_1154 (size=12304) 2024-11-15T14:15:15,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731680115383, max timestamp=1731680115477Num uniques 859; 2024-11-15T14:15:15,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T14:15:15,495 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/fc3f178d02094dc0ae728ffb4b79e504 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/fc3f178d02094dc0ae728ffb4b79e504 2024-11-15T14:15:15,502 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/A of 1fd20151ae970369f5abd5f0c2e6e70d into fc3f178d02094dc0ae728ffb4b79e504(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:15,502 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:15,502 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/A, priority=12, startTime=1731680115284; duration=0sec 2024-11-15T14:15:15,502 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:15,502 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:A 2024-11-15T14:15:15,503 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:15,506 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:15,506 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/C is initiating minor compaction (all files) 2024-11-15T14:15:15,506 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/C in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:15,507 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/106a6752f6224de2b0dbcfe9e66aa157, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/dce8cadfd87b460cb8080027746187c2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1fc651ca16a84da19172d6d6dc993389, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/42adeb719b0346a4ae36cc365927e12a] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=47.1 K 2024-11-15T14:15:15,507 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 106a6752f6224de2b0dbcfe9e66aa157, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731680110885 2024-11-15T14:15:15,511 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting dce8cadfd87b460cb8080027746187c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731680111470 2024-11-15T14:15:15,512 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fc651ca16a84da19172d6d6dc993389, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731680112157 2024-11-15T14:15:15,512 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42adeb719b0346a4ae36cc365927e12a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731680113484 2024-11-15T14:15:15,550 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:15,551 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/7dcd4d1ba9634da89964639c8261f62e is 50, key is test_row_0/C:col10/1731680113484/Put/seqid=0 2024-11-15T14:15:15,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741979_1155 (size=12391) 2024-11-15T14:15:15,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680175604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:15,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680175604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T14:15:15,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680175723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:15,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:15,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680175723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:15,788 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/87384dda24c342fabfc8199beff79ecf as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/87384dda24c342fabfc8199beff79ecf 2024-11-15T14:15:15,800 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/B of 1fd20151ae970369f5abd5f0c2e6e70d into 87384dda24c342fabfc8199beff79ecf(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:15,800 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:15,800 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/B, priority=12, startTime=1731680115284; duration=0sec 2024-11-15T14:15:15,801 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:15,801 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:B 2024-11-15T14:15:15,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:15,895 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155490cd06c03f4773b81b2e23ca1222cd_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155490cd06c03f4773b81b2e23ca1222cd_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/3cbfe2fd14ab4939aa26ba845624e36f, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/3cbfe2fd14ab4939aa26ba845624e36f is 175, key is test_row_0/A:col10/1731680114195/Put/seqid=0 2024-11-15T14:15:15,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741980_1156 (size=31105) 2024-11-15T14:15:15,943 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/3cbfe2fd14ab4939aa26ba845624e36f 2024-11-15T14:15:15,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680175941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:15,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680175942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/cc4ea404733049168a4c0e9666c0067a is 50, key is test_row_0/B:col10/1731680114195/Put/seqid=0 2024-11-15T14:15:15,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741981_1157 (size=12151) 2024-11-15T14:15:16,001 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/7dcd4d1ba9634da89964639c8261f62e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/7dcd4d1ba9634da89964639c8261f62e 2024-11-15T14:15:16,008 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/C of 1fd20151ae970369f5abd5f0c2e6e70d into 7dcd4d1ba9634da89964639c8261f62e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:16,008 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:16,008 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/C, priority=12, startTime=1731680115288; duration=0sec 2024-11-15T14:15:16,008 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:16,008 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:C 2024-11-15T14:15:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T14:15:16,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680176252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:16,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:16,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:16,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:16,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680176262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:16,401 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/cc4ea404733049168a4c0e9666c0067a 2024-11-15T14:15:16,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/ce4d13d2ed504144b818794c1cdfda8f is 50, key is test_row_0/C:col10/1731680114195/Put/seqid=0 2024-11-15T14:15:16,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741982_1158 (size=12151) 2024-11-15T14:15:16,448 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/ce4d13d2ed504144b818794c1cdfda8f 2024-11-15T14:15:16,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/3cbfe2fd14ab4939aa26ba845624e36f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/3cbfe2fd14ab4939aa26ba845624e36f 2024-11-15T14:15:16,472 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/3cbfe2fd14ab4939aa26ba845624e36f, entries=150, sequenceid=155, filesize=30.4 K 2024-11-15T14:15:16,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/cc4ea404733049168a4c0e9666c0067a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/cc4ea404733049168a4c0e9666c0067a 2024-11-15T14:15:16,491 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/cc4ea404733049168a4c0e9666c0067a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T14:15:16,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/ce4d13d2ed504144b818794c1cdfda8f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/ce4d13d2ed504144b818794c1cdfda8f 2024-11-15T14:15:16,501 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/ce4d13d2ed504144b818794c1cdfda8f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T14:15:16,503 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 1fd20151ae970369f5abd5f0c2e6e70d in 1138ms, sequenceid=155, compaction requested=false 2024-11-15T14:15:16,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:16,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:16,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-15T14:15:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-15T14:15:16,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-15T14:15:16,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4470 sec 2024-11-15T14:15:16,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4590 sec 2024-11-15T14:15:16,558 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T14:15:16,558 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T14:15:16,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-15T14:15:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:16,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:16,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:16,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:16,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:16,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:16,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:16,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115271b049b5fb5462987b67305cb2bb7d3_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680115597/Put/seqid=0 2024-11-15T14:15:16,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680176804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:16,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680176804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:16,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741983_1159 (size=14794) 2024-11-15T14:15:16,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680176922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:16,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680176922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:16,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:16,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:16,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:16,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:16,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:16,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680177142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:17,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680177142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T14:15:17,202 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:17,203 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-15T14:15:17,206 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:17,207 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:17,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T14:15:17,251 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:17,260 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115271b049b5fb5462987b67305cb2bb7d3_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115271b049b5fb5462987b67305cb2bb7d3_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:17,262 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/30faf6e2a6dc456c90ced667cd5561ca, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:17,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/30faf6e2a6dc456c90ced667cd5561ca is 175, key is test_row_0/A:col10/1731680115597/Put/seqid=0 2024-11-15T14:15:17,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741984_1160 (size=39749) 2024-11-15T14:15:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T14:15:17,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T14:15:17,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:17,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:17,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:17,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:17,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680177452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:17,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680177452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:17,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T14:15:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:17,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:17,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:17,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:17,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T14:15:17,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T14:15:17,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:17,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:17,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:17,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:17,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:17,705 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/30faf6e2a6dc456c90ced667cd5561ca 2024-11-15T14:15:17,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/75a25df2f70e4acba70b82ed160d9f63 is 50, key is test_row_0/B:col10/1731680115597/Put/seqid=0 2024-11-15T14:15:17,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741985_1161 (size=12151) 2024-11-15T14:15:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T14:15:17,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T14:15:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:17,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:17,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680177962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:17,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680177962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:18,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T14:15:18,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:18,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:18,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:18,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:18,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:18,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33914 deadline: 1731680178032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:18,033 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in 1fd20151ae970369f5abd5f0c2e6e70d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:18,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731680178062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:18,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731680178062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:18,068 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in 1fd20151ae970369f5abd5f0c2e6e70d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8319 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:18,070 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in 1fd20151ae970369f5abd5f0c2e6e70d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8315 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:18,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T14:15:18,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:18,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:18,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:18,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:18,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:18,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/75a25df2f70e4acba70b82ed160d9f63 2024-11-15T14:15:18,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/0a67082a13b045cb86df942d8c0b93d3 is 50, key is test_row_0/C:col10/1731680115597/Put/seqid=0 2024-11-15T14:15:18,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741986_1162 (size=12151) 2024-11-15T14:15:18,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/0a67082a13b045cb86df942d8c0b93d3 2024-11-15T14:15:18,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/30faf6e2a6dc456c90ced667cd5561ca as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/30faf6e2a6dc456c90ced667cd5561ca 2024-11-15T14:15:18,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/30faf6e2a6dc456c90ced667cd5561ca, entries=200, sequenceid=195, filesize=38.8 K 2024-11-15T14:15:18,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/75a25df2f70e4acba70b82ed160d9f63 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/75a25df2f70e4acba70b82ed160d9f63 2024-11-15T14:15:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/75a25df2f70e4acba70b82ed160d9f63, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T14:15:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/0a67082a13b045cb86df942d8c0b93d3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/0a67082a13b045cb86df942d8c0b93d3 2024-11-15T14:15:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T14:15:18,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:18,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:18,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:18,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:18,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/0a67082a13b045cb86df942d8c0b93d3, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T14:15:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=40.25 KB/41220 for 1fd20151ae970369f5abd5f0c2e6e70d in 1544ms, sequenceid=195, compaction requested=true 2024-11-15T14:15:18,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:18,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:18,321 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:18,321 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:18,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:18,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:18,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:18,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,323 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:18,323 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:18,323 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/B is initiating minor compaction (all files) 2024-11-15T14:15:18,323 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/A is initiating minor compaction (all files) 2024-11-15T14:15:18,324 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/B in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:18,324 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/A in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:18,324 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/87384dda24c342fabfc8199beff79ecf, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/cc4ea404733049168a4c0e9666c0067a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/75a25df2f70e4acba70b82ed160d9f63] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=35.8 K 2024-11-15T14:15:18,324 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/fc3f178d02094dc0ae728ffb4b79e504, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/3cbfe2fd14ab4939aa26ba845624e36f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/30faf6e2a6dc456c90ced667cd5561ca] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=99.8 K 2024-11-15T14:15:18,324 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:18,324 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/fc3f178d02094dc0ae728ffb4b79e504, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/3cbfe2fd14ab4939aa26ba845624e36f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/30faf6e2a6dc456c90ced667cd5561ca] 2024-11-15T14:15:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,324 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc3f178d02094dc0ae728ffb4b79e504, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731680113484 2024-11-15T14:15:18,324 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 87384dda24c342fabfc8199beff79ecf, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731680113484 2024-11-15T14:15:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,326 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cbfe2fd14ab4939aa26ba845624e36f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731680114195 2024-11-15T14:15:18,326 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting cc4ea404733049168a4c0e9666c0067a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731680114195 2024-11-15T14:15:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,326 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30faf6e2a6dc456c90ced667cd5561ca, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731680115581 2024-11-15T14:15:18,327 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 75a25df2f70e4acba70b82ed160d9f63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731680115583 2024-11-15T14:15:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T14:15:18,352 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,365 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,365 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/f71d1074930d4f79a3ecdbf91890c913 is 50, key is test_row_0/B:col10/1731680115597/Put/seqid=0 2024-11-15T14:15:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,371 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411150bd51864484b4937857a0b81aecc6ab3_1fd20151ae970369f5abd5f0c2e6e70d store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,374 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411150bd51864484b4937857a0b81aecc6ab3_1fd20151ae970369f5abd5f0c2e6e70d, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:18,374 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150bd51864484b4937857a0b81aecc6ab3_1fd20151ae970369f5abd5f0c2e6e70d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741987_1163 (size=12493) 2024-11-15T14:15:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741988_1164 (size=4469) 2024-11-15T14:15:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,439 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#A#compaction#137 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,440 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/d413db03409947be813441ca2418f685 is 175, key is test_row_0/A:col10/1731680115597/Put/seqid=0 2024-11-15T14:15:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741989_1165 (size=31447) 2024-11-15T14:15:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T14:15:18,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:18,470 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-15T14:15:18,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:18,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:18,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:18,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:18,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:18,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bf1a4923820b4e6aa5e81e3444cffd3c_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_1/A:col10/1731680116802/Put/seqid=0 2024-11-15T14:15:18,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741990_1166 (size=9814) 2024-11-15T14:15:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,803 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/f71d1074930d4f79a3ecdbf91890c913 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f71d1074930d4f79a3ecdbf91890c913 2024-11-15T14:15:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,818 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/B of 1fd20151ae970369f5abd5f0c2e6e70d into f71d1074930d4f79a3ecdbf91890c913(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:18,818 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,818 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/B, priority=13, startTime=1731680118321; duration=0sec 2024-11-15T14:15:18,818 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:18,818 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:B 2024-11-15T14:15:18,819 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,823 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:18,823 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/C is initiating minor compaction (all files) 2024-11-15T14:15:18,823 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/C in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,823 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/7dcd4d1ba9634da89964639c8261f62e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/ce4d13d2ed504144b818794c1cdfda8f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/0a67082a13b045cb86df942d8c0b93d3] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=35.8 K 2024-11-15T14:15:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,829 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dcd4d1ba9634da89964639c8261f62e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731680113484 2024-11-15T14:15:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,831 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting ce4d13d2ed504144b818794c1cdfda8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731680114195 2024-11-15T14:15:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,831 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a67082a13b045cb86df942d8c0b93d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731680115583 2024-11-15T14:15:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,850 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:18,850 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/e3e991d5e583431787ba1eecd3c6c460 is 50, key is test_row_0/C:col10/1731680115597/Put/seqid=0 2024-11-15T14:15:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,871 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/d413db03409947be813441ca2418f685 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/d413db03409947be813441ca2418f685 2024-11-15T14:15:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,887 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/A of 1fd20151ae970369f5abd5f0c2e6e70d into d413db03409947be813441ca2418f685(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:18,887 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:18,887 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/A, priority=13, startTime=1731680118320; duration=0sec 2024-11-15T14:15:18,887 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:18,887 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:A 2024-11-15T14:15:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741991_1167 (size=12493) 2024-11-15T14:15:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,901 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/e3e991d5e583431787ba1eecd3c6c460 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/e3e991d5e583431787ba1eecd3c6c460 2024-11-15T14:15:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,913 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/C of 1fd20151ae970369f5abd5f0c2e6e70d into e3e991d5e583431787ba1eecd3c6c460(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:18,913 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:18,913 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/C, priority=13, startTime=1731680118321; duration=0sec 2024-11-15T14:15:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,913 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:18,913 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:C 2024-11-15T14:15:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,928 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bf1a4923820b4e6aa5e81e3444cffd3c_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf1a4923820b4e6aa5e81e3444cffd3c_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/2cfdf6bcff0c4c9891a1ac8d23a9096c, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/2cfdf6bcff0c4c9891a1ac8d23a9096c is 175, key is test_row_1/A:col10/1731680116802/Put/seqid=0 2024-11-15T14:15:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741992_1168 (size=22461) 2024-11-15T14:15:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,955 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/2cfdf6bcff0c4c9891a1ac8d23a9096c 2024-11-15T14:15:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/6e21dd28a53d49ca9c68f2d777fc4e71 is 50, key is test_row_1/B:col10/1731680116802/Put/seqid=0 2024-11-15T14:15:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741993_1169 (size=9757) 2024-11-15T14:15:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680119054, max timestamp=1731680119101Num uniques 859; 2024-11-15T14:15:19,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:15:19,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680119073, max timestamp=1731680119104Num uniques 859; 2024-11-15T14:15:19,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680119073, max timestamp=1731680119101Num uniques 859; 2024-11-15T14:15:19,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:19,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:19,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731680119126, max timestamp=1731680119164Num uniques 1068; 2024-11-15T14:15:19,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731680119126, max timestamp=1731680119164Num uniques 1018; 2024-11-15T14:15:19,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-15T14:15:19,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-15T14:15:19,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731680119126, max timestamp=1731680119164Num uniques 1018; 2024-11-15T14:15:19,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-15T14:15:19,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731680119223, max timestamp=1731680119274Num uniques 1027; 2024-11-15T14:15:19,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-15T14:15:19,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731680119223, max timestamp=1731680119274Num uniques 1027; 2024-11-15T14:15:19,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-15T14:15:19,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731680119223, max timestamp=1731680119274Num uniques 1027; 2024-11-15T14:15:19,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-15T14:15:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T14:15:19,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731680119333, max timestamp=1731680119400Num uniques 1036; 2024-11-15T14:15:19,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-15T14:15:19,417 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/6e21dd28a53d49ca9c68f2d777fc4e71 2024-11-15T14:15:19,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731680119333, max timestamp=1731680119400Num uniques 1036; 2024-11-15T14:15:19,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731680119333, max timestamp=1731680119400Num uniques 1036; 2024-11-15T14:15:19,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-15T14:15:19,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-15T14:15:19,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/a208f2e625c046199d95d7b151f9fac6 is 50, key is test_row_1/C:col10/1731680116802/Put/seqid=0 2024-11-15T14:15:19,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741994_1170 (size=9757) 2024-11-15T14:15:19,461 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/a208f2e625c046199d95d7b151f9fac6 2024-11-15T14:15:19,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/2cfdf6bcff0c4c9891a1ac8d23a9096c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2cfdf6bcff0c4c9891a1ac8d23a9096c 2024-11-15T14:15:19,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1731680119448, max timestamp=1731680119477Num uniques 1095; 2024-11-15T14:15:19,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50140, new segment data size=8920, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-15T14:15:19,485 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2cfdf6bcff0c4c9891a1ac8d23a9096c, entries=100, sequenceid=204, filesize=21.9 K 2024-11-15T14:15:19,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731680119448, max timestamp=1731680119476Num uniques 1045; 2024-11-15T14:15:19,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-15T14:15:19,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731680119448, max timestamp=1731680119476Num uniques 1045; 2024-11-15T14:15:19,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-15T14:15:19,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/6e21dd28a53d49ca9c68f2d777fc4e71 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/6e21dd28a53d49ca9c68f2d777fc4e71 2024-11-15T14:15:19,504 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/6e21dd28a53d49ca9c68f2d777fc4e71, entries=100, sequenceid=204, filesize=9.5 K 2024-11-15T14:15:19,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/a208f2e625c046199d95d7b151f9fac6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/a208f2e625c046199d95d7b151f9fac6 2024-11-15T14:15:19,515 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/a208f2e625c046199d95d7b151f9fac6, entries=100, sequenceid=204, filesize=9.5 K 2024-11-15T14:15:19,517 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=67.09 KB/68700 for 1fd20151ae970369f5abd5f0c2e6e70d in 1046ms, sequenceid=204, compaction requested=false 2024-11-15T14:15:19,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:19,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:19,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-15T14:15:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-15T14:15:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:19,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=73.80 KB heapSize=202.31 KB 2024-11-15T14:15:19,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:19,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:19,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:19,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:19,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:19,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:19,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-15T14:15:19,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3120 sec 2024-11-15T14:15:19,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.3200 sec 2024-11-15T14:15:19,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fa4cafdde1f7479bbd40c34ea8750fa3_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680119467/Put/seqid=0 2024-11-15T14:15:19,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741995_1171 (size=17534) 2024-11-15T14:15:19,589 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:19,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680179583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:19,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680179584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:19,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:19,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:19,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:19,595 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fa4cafdde1f7479bbd40c34ea8750fa3_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fa4cafdde1f7479bbd40c34ea8750fa3_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:19,596 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/0064f858fcb24b3b80e0c2f6627ea137, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:19,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/0064f858fcb24b3b80e0c2f6627ea137 is 175, key is test_row_0/A:col10/1731680119467/Put/seqid=0 2024-11-15T14:15:19,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741996_1172 (size=48639) 2024-11-15T14:15:19,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680179702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:19,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680179702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:19,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680179912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:19,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680179924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:19,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:19,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:19,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:20,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/0064f858fcb24b3b80e0c2f6627ea137 2024-11-15T14:15:20,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/9b10832fbbae47cb98fc6825629aaa02 is 50, key is test_row_0/B:col10/1731680119467/Put/seqid=0 2024-11-15T14:15:20,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741997_1173 (size=12301) 2024-11-15T14:15:20,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680180232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:20,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680180244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:20,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:20,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:20,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:20,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/9b10832fbbae47cb98fc6825629aaa02 2024-11-15T14:15:20,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/9765a483bd314b0292bb6d933e4b52be is 50, key is test_row_0/C:col10/1731680119467/Put/seqid=0 2024-11-15T14:15:20,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741998_1174 (size=12301) 2024-11-15T14:15:20,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:20,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680180752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680180752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:20,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/9765a483bd314b0292bb6d933e4b52be 2024-11-15T14:15:20,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/0064f858fcb24b3b80e0c2f6627ea137 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0064f858fcb24b3b80e0c2f6627ea137 2024-11-15T14:15:20,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0064f858fcb24b3b80e0c2f6627ea137, entries=250, sequenceid=303, filesize=47.5 K 2024-11-15T14:15:20,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/9b10832fbbae47cb98fc6825629aaa02 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9b10832fbbae47cb98fc6825629aaa02 2024-11-15T14:15:21,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9b10832fbbae47cb98fc6825629aaa02, entries=150, sequenceid=303, filesize=12.0 K 2024-11-15T14:15:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/9765a483bd314b0292bb6d933e4b52be as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/9765a483bd314b0292bb6d933e4b52be 2024-11-15T14:15:21,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/9765a483bd314b0292bb6d933e4b52be, entries=150, sequenceid=303, filesize=12.0 K 2024-11-15T14:15:21,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~202.27 KB/207120, currentSize=107.34 KB/109920 for 1fd20151ae970369f5abd5f0c2e6e70d in 1510ms, sequenceid=303, compaction requested=true 2024-11-15T14:15:21,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:21,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:21,031 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:21,036 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102547 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:21,036 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/A is initiating minor compaction (all files) 2024-11-15T14:15:21,036 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/A in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:21,037 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/d413db03409947be813441ca2418f685, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2cfdf6bcff0c4c9891a1ac8d23a9096c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0064f858fcb24b3b80e0c2f6627ea137] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=100.1 K 2024-11-15T14:15:21,037 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:21,037 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/d413db03409947be813441ca2418f685, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2cfdf6bcff0c4c9891a1ac8d23a9096c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0064f858fcb24b3b80e0c2f6627ea137] 2024-11-15T14:15:21,037 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting d413db03409947be813441ca2418f685, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731680115583 2024-11-15T14:15:21,038 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cfdf6bcff0c4c9891a1ac8d23a9096c, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731680116802 2024-11-15T14:15:21,038 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0064f858fcb24b3b80e0c2f6627ea137, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731680119467 2024-11-15T14:15:21,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:21,043 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:21,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:21,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:21,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:21,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:21,046 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:21,046 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/B is initiating minor compaction (all files) 2024-11-15T14:15:21,046 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/B in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:21,046 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f71d1074930d4f79a3ecdbf91890c913, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/6e21dd28a53d49ca9c68f2d777fc4e71, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9b10832fbbae47cb98fc6825629aaa02] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=33.7 K 2024-11-15T14:15:21,047 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting f71d1074930d4f79a3ecdbf91890c913, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731680115583 2024-11-15T14:15:21,047 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e21dd28a53d49ca9c68f2d777fc4e71, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731680116802 2024-11-15T14:15:21,048 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b10832fbbae47cb98fc6825629aaa02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731680119467 2024-11-15T14:15:21,071 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:21,083 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115e410862f8b374b8086aad8e2bc0b7a3f_1fd20151ae970369f5abd5f0c2e6e70d store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:21,083 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:21,084 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/460487d1926640cea3c413aec36604f2 is 50, key is test_row_0/B:col10/1731680119467/Put/seqid=0 2024-11-15T14:15:21,085 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115e410862f8b374b8086aad8e2bc0b7a3f_1fd20151ae970369f5abd5f0c2e6e70d, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:21,085 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e410862f8b374b8086aad8e2bc0b7a3f_1fd20151ae970369f5abd5f0c2e6e70d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:21,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741999_1175 (size=12745) 2024-11-15T14:15:21,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742000_1176 (size=4469) 2024-11-15T14:15:21,149 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#A#compaction#146 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:21,150 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/893801de8a6b456792d7127ffa079dd3 is 175, key is test_row_0/A:col10/1731680119467/Put/seqid=0 2024-11-15T14:15:21,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742001_1177 (size=31699) 2024-11-15T14:15:21,212 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/893801de8a6b456792d7127ffa079dd3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/893801de8a6b456792d7127ffa079dd3 2024-11-15T14:15:21,221 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/A of 1fd20151ae970369f5abd5f0c2e6e70d into 893801de8a6b456792d7127ffa079dd3(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:21,221 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:21,221 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/A, priority=13, startTime=1731680121031; duration=0sec 2024-11-15T14:15:21,221 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:21,221 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:A 2024-11-15T14:15:21,221 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:21,223 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:21,223 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/C is initiating minor compaction (all files) 2024-11-15T14:15:21,223 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/C in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:21,224 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/e3e991d5e583431787ba1eecd3c6c460, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/a208f2e625c046199d95d7b151f9fac6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/9765a483bd314b0292bb6d933e4b52be] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=33.7 K 2024-11-15T14:15:21,225 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3e991d5e583431787ba1eecd3c6c460, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731680115583 2024-11-15T14:15:21,225 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a208f2e625c046199d95d7b151f9fac6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731680116802 2024-11-15T14:15:21,226 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9765a483bd314b0292bb6d933e4b52be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731680119467 2024-11-15T14:15:21,239 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#C#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:21,255 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/9c810afd43e841c38cf1f049172e177b is 50, key is test_row_0/C:col10/1731680119467/Put/seqid=0 2024-11-15T14:15:21,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742002_1178 (size=12745) 2024-11-15T14:15:21,303 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/9c810afd43e841c38cf1f049172e177b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/9c810afd43e841c38cf1f049172e177b 2024-11-15T14:15:21,314 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/C of 1fd20151ae970369f5abd5f0c2e6e70d into 9c810afd43e841c38cf1f049172e177b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:21,314 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:21,314 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/C, priority=13, startTime=1731680121045; duration=0sec 2024-11-15T14:15:21,314 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:21,314 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:C 2024-11-15T14:15:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T14:15:21,374 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:21,375 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-15T14:15:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T14:15:21,378 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:21,380 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:21,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T14:15:21,529 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/460487d1926640cea3c413aec36604f2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/460487d1926640cea3c413aec36604f2 2024-11-15T14:15:21,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T14:15:21,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:21,533 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T14:15:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:21,542 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/B of 1fd20151ae970369f5abd5f0c2e6e70d into 460487d1926640cea3c413aec36604f2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:21,542 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:21,542 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/B, priority=13, startTime=1731680121043; duration=0sec 2024-11-15T14:15:21,542 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:21,542 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:B 2024-11-15T14:15:21,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158055612172304d2ea9c470a2e0459194_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680119571/Put/seqid=0 2024-11-15T14:15:21,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742003_1179 (size=12454) 2024-11-15T14:15:21,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:21,611 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158055612172304d2ea9c470a2e0459194_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158055612172304d2ea9c470a2e0459194_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:21,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/e2712b3287d04eaea8ec57c8be475d41, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:21,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/e2712b3287d04eaea8ec57c8be475d41 is 175, key is test_row_0/A:col10/1731680119571/Put/seqid=0 2024-11-15T14:15:21,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742004_1180 (size=31255) 2024-11-15T14:15:21,658 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/e2712b3287d04eaea8ec57c8be475d41 2024-11-15T14:15:21,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/828584426f64424d9433df2357e3111a is 50, key is test_row_0/B:col10/1731680119571/Put/seqid=0 2024-11-15T14:15:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T14:15:21,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742005_1181 (size=12301) 2024-11-15T14:15:21,728 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/828584426f64424d9433df2357e3111a 2024-11-15T14:15:21,730 INFO [master/26754e923c4a:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-15T14:15:21,730 INFO [master/26754e923c4a:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-15T14:15:21,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/585c472ae5964df1b72ae3330927a78d is 50, key is test_row_0/C:col10/1731680119571/Put/seqid=0 2024-11-15T14:15:21,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:21,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742006_1182 (size=12301) 2024-11-15T14:15:21,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680181803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:21,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680181811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:21,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680181922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:21,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680181922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:21,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T14:15:22,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680182132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:22,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680182132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:22,193 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/585c472ae5964df1b72ae3330927a78d 2024-11-15T14:15:22,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/e2712b3287d04eaea8ec57c8be475d41 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/e2712b3287d04eaea8ec57c8be475d41 2024-11-15T14:15:22,209 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/e2712b3287d04eaea8ec57c8be475d41, entries=150, sequenceid=324, filesize=30.5 K 2024-11-15T14:15:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/828584426f64424d9433df2357e3111a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/828584426f64424d9433df2357e3111a 2024-11-15T14:15:22,221 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/828584426f64424d9433df2357e3111a, entries=150, sequenceid=324, filesize=12.0 K 2024-11-15T14:15:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/585c472ae5964df1b72ae3330927a78d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/585c472ae5964df1b72ae3330927a78d 2024-11-15T14:15:22,229 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/585c472ae5964df1b72ae3330927a78d, entries=150, sequenceid=324, filesize=12.0 K 2024-11-15T14:15:22,232 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1fd20151ae970369f5abd5f0c2e6e70d in 698ms, sequenceid=324, compaction requested=false 2024-11-15T14:15:22,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:22,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:22,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-15T14:15:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-15T14:15:22,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-15T14:15:22,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 854 msec 2024-11-15T14:15:22,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 862 msec 2024-11-15T14:15:22,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T14:15:22,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:22,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:22,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:22,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:22,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:22,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:22,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150926f411a2824d7f883978121b3decf8_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680122444/Put/seqid=0 2024-11-15T14:15:22,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742007_1183 (size=12454) 2024-11-15T14:15:22,491 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:22,500 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150926f411a2824d7f883978121b3decf8_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150926f411a2824d7f883978121b3decf8_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:22,502 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/2325c8db66e544d5abab5b7ae96ba2e5, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:22,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/2325c8db66e544d5abab5b7ae96ba2e5 is 175, key is test_row_0/A:col10/1731680122444/Put/seqid=0 2024-11-15T14:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T14:15:22,512 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:22,512 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-15T14:15:22,515 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:22,517 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:22,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T14:15:22,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742008_1184 (size=31255) 2024-11-15T14:15:22,528 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/2325c8db66e544d5abab5b7ae96ba2e5 2024-11-15T14:15:22,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680182532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:22,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:22,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680182541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:22,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/5ad459c938814ac2920cf8db8e41f974 is 50, key is test_row_0/B:col10/1731680122444/Put/seqid=0 2024-11-15T14:15:22,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742009_1185 (size=12301) 2024-11-15T14:15:22,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/5ad459c938814ac2920cf8db8e41f974 2024-11-15T14:15:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T14:15:22,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/7b745ea9ca284da1940eea7c83938797 is 50, key is test_row_0/C:col10/1731680122444/Put/seqid=0 2024-11-15T14:15:22,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680182651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:22,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680182651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:22,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:22,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742010_1186 (size=12301) 2024-11-15T14:15:22,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/7b745ea9ca284da1940eea7c83938797 2024-11-15T14:15:22,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T14:15:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:22,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:22,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:22,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/2325c8db66e544d5abab5b7ae96ba2e5 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2325c8db66e544d5abab5b7ae96ba2e5 2024-11-15T14:15:22,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2325c8db66e544d5abab5b7ae96ba2e5, entries=150, sequenceid=340, filesize=30.5 K 2024-11-15T14:15:22,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/5ad459c938814ac2920cf8db8e41f974 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/5ad459c938814ac2920cf8db8e41f974 2024-11-15T14:15:22,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/5ad459c938814ac2920cf8db8e41f974, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T14:15:22,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/7b745ea9ca284da1940eea7c83938797 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/7b745ea9ca284da1940eea7c83938797 2024-11-15T14:15:22,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/7b745ea9ca284da1940eea7c83938797, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T14:15:22,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1fd20151ae970369f5abd5f0c2e6e70d in 262ms, sequenceid=340, compaction requested=true 2024-11-15T14:15:22,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:22,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:22,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:22,715 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:22,715 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:22,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:22,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:22,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:22,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:22,717 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:22,717 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/A is initiating minor compaction (all files) 2024-11-15T14:15:22,717 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/A in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:22,717 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/893801de8a6b456792d7127ffa079dd3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/e2712b3287d04eaea8ec57c8be475d41, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2325c8db66e544d5abab5b7ae96ba2e5] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=92.0 K 2024-11-15T14:15:22,717 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:22,717 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/893801de8a6b456792d7127ffa079dd3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/e2712b3287d04eaea8ec57c8be475d41, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2325c8db66e544d5abab5b7ae96ba2e5] 2024-11-15T14:15:22,718 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:22,718 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/B is initiating minor compaction (all files) 2024-11-15T14:15:22,718 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/B in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:22,718 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/460487d1926640cea3c413aec36604f2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/828584426f64424d9433df2357e3111a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/5ad459c938814ac2920cf8db8e41f974] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=36.5 K 2024-11-15T14:15:22,718 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 893801de8a6b456792d7127ffa079dd3, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731680119467 2024-11-15T14:15:22,718 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2712b3287d04eaea8ec57c8be475d41, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731680119571 2024-11-15T14:15:22,719 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 460487d1926640cea3c413aec36604f2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731680119467 2024-11-15T14:15:22,719 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 828584426f64424d9433df2357e3111a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731680119571 2024-11-15T14:15:22,719 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2325c8db66e544d5abab5b7ae96ba2e5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731680121793 2024-11-15T14:15:22,720 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ad459c938814ac2920cf8db8e41f974, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731680121793 2024-11-15T14:15:22,741 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#B#compaction#155 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:22,742 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/f7e3bfd1d3d54b61bc3471575d4151c6 is 50, key is test_row_0/B:col10/1731680122444/Put/seqid=0 2024-11-15T14:15:22,751 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:22,768 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411154220d6df8c974979a5e5906a3c7c9ea0_1fd20151ae970369f5abd5f0c2e6e70d store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:22,770 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411154220d6df8c974979a5e5906a3c7c9ea0_1fd20151ae970369f5abd5f0c2e6e70d, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:22,770 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154220d6df8c974979a5e5906a3c7c9ea0_1fd20151ae970369f5abd5f0c2e6e70d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:22,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742011_1187 (size=12847) 2024-11-15T14:15:22,797 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/f7e3bfd1d3d54b61bc3471575d4151c6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f7e3bfd1d3d54b61bc3471575d4151c6 2024-11-15T14:15:22,808 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/B of 1fd20151ae970369f5abd5f0c2e6e70d into f7e3bfd1d3d54b61bc3471575d4151c6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:22,808 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:22,808 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/B, priority=13, startTime=1731680122715; duration=0sec 2024-11-15T14:15:22,808 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:22,808 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:B 2024-11-15T14:15:22,808 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:22,810 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:22,810 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/C is initiating minor compaction (all files) 2024-11-15T14:15:22,810 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/C in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:22,810 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/9c810afd43e841c38cf1f049172e177b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/585c472ae5964df1b72ae3330927a78d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/7b745ea9ca284da1940eea7c83938797] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=36.5 K 2024-11-15T14:15:22,811 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c810afd43e841c38cf1f049172e177b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731680119467 2024-11-15T14:15:22,812 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 585c472ae5964df1b72ae3330927a78d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731680119571 2024-11-15T14:15:22,812 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b745ea9ca284da1940eea7c83938797, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731680121793 2024-11-15T14:15:22,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T14:15:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:22,826 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T14:15:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T14:15:22,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742012_1188 (size=4469) 2024-11-15T14:15:22,844 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#C#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:22,845 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/844c4668a1504cf3991eac56d736e821 is 50, key is test_row_0/C:col10/1731680122444/Put/seqid=0 2024-11-15T14:15:22,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157ce9d266cfef4f37988c9d0c0fa90ad5_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680122521/Put/seqid=0 2024-11-15T14:15:22,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:22,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742013_1189 (size=12847) 2024-11-15T14:15:22,893 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/844c4668a1504cf3991eac56d736e821 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/844c4668a1504cf3991eac56d736e821 2024-11-15T14:15:22,907 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/C of 1fd20151ae970369f5abd5f0c2e6e70d into 844c4668a1504cf3991eac56d736e821(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:22,907 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:22,907 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/C, priority=13, startTime=1731680122716; duration=0sec 2024-11-15T14:15:22,907 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:22,907 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:C 2024-11-15T14:15:22,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742014_1190 (size=12454) 2024-11-15T14:15:22,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:22,927 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157ce9d266cfef4f37988c9d0c0fa90ad5_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157ce9d266cfef4f37988c9d0c0fa90ad5_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:22,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/4541c3f2254a493497de5e454902c27d, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:22,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/4541c3f2254a493497de5e454902c27d is 175, key is test_row_0/A:col10/1731680122521/Put/seqid=0 2024-11-15T14:15:22,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680182934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:22,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680182937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:22,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742015_1191 (size=31255) 2024-11-15T14:15:23,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680183043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:23,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680183052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:23,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T14:15:23,240 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#A#compaction#156 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:23,241 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/366b62c6698246bd80526a606b812e18 is 175, key is test_row_0/A:col10/1731680122444/Put/seqid=0 2024-11-15T14:15:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680183262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:23,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680183272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:23,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742016_1192 (size=31801) 2024-11-15T14:15:23,362 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/4541c3f2254a493497de5e454902c27d 2024-11-15T14:15:23,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/a0ff0fa4d418442086ccb8391459fec7 is 50, key is test_row_0/B:col10/1731680122521/Put/seqid=0 2024-11-15T14:15:23,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742017_1193 (size=12301) 2024-11-15T14:15:23,431 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/a0ff0fa4d418442086ccb8391459fec7 2024-11-15T14:15:23,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/1775728b58974f3996ed9c2d8b001776 is 50, key is test_row_0/C:col10/1731680122521/Put/seqid=0 2024-11-15T14:15:23,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742018_1194 (size=12301) 2024-11-15T14:15:23,514 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/1775728b58974f3996ed9c2d8b001776 2024-11-15T14:15:23,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/4541c3f2254a493497de5e454902c27d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4541c3f2254a493497de5e454902c27d 2024-11-15T14:15:23,536 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4541c3f2254a493497de5e454902c27d, entries=150, sequenceid=358, filesize=30.5 K 2024-11-15T14:15:23,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/a0ff0fa4d418442086ccb8391459fec7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a0ff0fa4d418442086ccb8391459fec7 2024-11-15T14:15:23,545 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a0ff0fa4d418442086ccb8391459fec7, entries=150, sequenceid=358, filesize=12.0 K 2024-11-15T14:15:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/1775728b58974f3996ed9c2d8b001776 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1775728b58974f3996ed9c2d8b001776 2024-11-15T14:15:23,552 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1775728b58974f3996ed9c2d8b001776, entries=150, sequenceid=358, filesize=12.0 K 2024-11-15T14:15:23,554 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1fd20151ae970369f5abd5f0c2e6e70d in 728ms, sequenceid=358, compaction requested=false 2024-11-15T14:15:23,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:23,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:23,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-15T14:15:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-15T14:15:23,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-15T14:15:23,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0390 sec 2024-11-15T14:15:23,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.0470 sec 2024-11-15T14:15:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:23,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T14:15:23,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:23,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:23,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:23,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:23,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:23,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:23,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158b4b52362311492b93fd22acd4e568da_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680122935/Put/seqid=0 2024-11-15T14:15:23,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680183644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:23,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742019_1195 (size=14994) 2024-11-15T14:15:23,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680183645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T14:15:23,652 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:23,652 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:23,653 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-15T14:15:23,655 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:23,657 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T14:15:23,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:23,665 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158b4b52362311492b93fd22acd4e568da_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158b4b52362311492b93fd22acd4e568da_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:23,667 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/cc5097831abb451b8e933dfd902d6ddc, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:23,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/cc5097831abb451b8e933dfd902d6ddc is 175, key is test_row_0/A:col10/1731680122935/Put/seqid=0 2024-11-15T14:15:23,704 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/366b62c6698246bd80526a606b812e18 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/366b62c6698246bd80526a606b812e18 2024-11-15T14:15:23,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742020_1196 (size=39949) 2024-11-15T14:15:23,712 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/A of 1fd20151ae970369f5abd5f0c2e6e70d into 366b62c6698246bd80526a606b812e18(size=31.1 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:23,712 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:23,712 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/A, priority=13, startTime=1731680122715; duration=0sec 2024-11-15T14:15:23,712 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:23,712 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:A 2024-11-15T14:15:23,714 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/cc5097831abb451b8e933dfd902d6ddc 2024-11-15T14:15:23,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/54958d95d0404a76b911880af1af9a00 is 50, key is test_row_0/B:col10/1731680122935/Put/seqid=0 2024-11-15T14:15:23,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680183757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T14:15:23,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680183762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:23,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742021_1197 (size=12301) 2024-11-15T14:15:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:23,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/54958d95d0404a76b911880af1af9a00 2024-11-15T14:15:23,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/5585a0ef670d435c8b089f0497458c33 is 50, key is test_row_0/C:col10/1731680122935/Put/seqid=0 2024-11-15T14:15:23,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T14:15:23,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:23,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:23,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:23,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:23,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:23,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742022_1198 (size=12301) 2024-11-15T14:15:23,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/5585a0ef670d435c8b089f0497458c33 2024-11-15T14:15:23,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/cc5097831abb451b8e933dfd902d6ddc as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/cc5097831abb451b8e933dfd902d6ddc 2024-11-15T14:15:23,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/cc5097831abb451b8e933dfd902d6ddc, entries=200, sequenceid=375, filesize=39.0 K 2024-11-15T14:15:23,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/54958d95d0404a76b911880af1af9a00 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/54958d95d0404a76b911880af1af9a00 2024-11-15T14:15:23,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/54958d95d0404a76b911880af1af9a00, entries=150, sequenceid=375, filesize=12.0 K 2024-11-15T14:15:23,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/5585a0ef670d435c8b089f0497458c33 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/5585a0ef670d435c8b089f0497458c33 2024-11-15T14:15:23,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/5585a0ef670d435c8b089f0497458c33, entries=150, sequenceid=375, filesize=12.0 K 2024-11-15T14:15:23,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 1fd20151ae970369f5abd5f0c2e6e70d in 283ms, sequenceid=375, compaction requested=true 2024-11-15T14:15:23,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:23,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:23,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:23,863 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:23,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:23,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:23,863 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:23,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:23,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:23,864 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:23,864 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/B is initiating minor compaction (all files) 2024-11-15T14:15:23,865 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:23,865 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/B in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:23,865 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/A is initiating minor compaction (all files) 2024-11-15T14:15:23,865 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f7e3bfd1d3d54b61bc3471575d4151c6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a0ff0fa4d418442086ccb8391459fec7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/54958d95d0404a76b911880af1af9a00] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=36.6 K 2024-11-15T14:15:23,865 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/A in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:23,865 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/366b62c6698246bd80526a606b812e18, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4541c3f2254a493497de5e454902c27d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/cc5097831abb451b8e933dfd902d6ddc] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=100.6 K 2024-11-15T14:15:23,865 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:23,865 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/366b62c6698246bd80526a606b812e18, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4541c3f2254a493497de5e454902c27d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/cc5097831abb451b8e933dfd902d6ddc] 2024-11-15T14:15:23,865 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e3bfd1d3d54b61bc3471575d4151c6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731680121793 2024-11-15T14:15:23,866 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 366b62c6698246bd80526a606b812e18, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731680121793 2024-11-15T14:15:23,866 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ff0fa4d418442086ccb8391459fec7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731680122487 2024-11-15T14:15:23,866 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 54958d95d0404a76b911880af1af9a00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731680122923 2024-11-15T14:15:23,866 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4541c3f2254a493497de5e454902c27d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731680122487 2024-11-15T14:15:23,867 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc5097831abb451b8e933dfd902d6ddc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731680122923 2024-11-15T14:15:23,891 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#B#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:23,891 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/19feb7a465ba4f83a155ba8e99aa1b3a is 50, key is test_row_0/B:col10/1731680122935/Put/seqid=0 2024-11-15T14:15:23,895 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:23,903 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115c2226466d3704609ac4e9e3605cc47a4_1fd20151ae970369f5abd5f0c2e6e70d store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:23,906 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115c2226466d3704609ac4e9e3605cc47a4_1fd20151ae970369f5abd5f0c2e6e70d, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:23,906 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c2226466d3704609ac4e9e3605cc47a4_1fd20151ae970369f5abd5f0c2e6e70d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:23,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742023_1199 (size=12949) 2024-11-15T14:15:23,946 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/19feb7a465ba4f83a155ba8e99aa1b3a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/19feb7a465ba4f83a155ba8e99aa1b3a 2024-11-15T14:15:23,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742024_1200 (size=4469) 2024-11-15T14:15:23,953 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#A#compaction#165 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:23,953 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/0ca06830d6904d988b27e0e6a0ec2d20 is 175, key is test_row_0/A:col10/1731680122935/Put/seqid=0 2024-11-15T14:15:23,954 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/B of 1fd20151ae970369f5abd5f0c2e6e70d into 19feb7a465ba4f83a155ba8e99aa1b3a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:23,954 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:23,954 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/B, priority=13, startTime=1731680123863; duration=0sec 2024-11-15T14:15:23,954 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:23,954 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:B 2024-11-15T14:15:23,954 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:23,957 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:23,957 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/C is initiating minor compaction (all files) 2024-11-15T14:15:23,957 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/C in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:23,957 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/844c4668a1504cf3991eac56d736e821, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1775728b58974f3996ed9c2d8b001776, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/5585a0ef670d435c8b089f0497458c33] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=36.6 K 2024-11-15T14:15:23,958 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 844c4668a1504cf3991eac56d736e821, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731680121793 2024-11-15T14:15:23,958 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1775728b58974f3996ed9c2d8b001776, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731680122487 2024-11-15T14:15:23,959 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5585a0ef670d435c8b089f0497458c33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731680122923 2024-11-15T14:15:23,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T14:15:23,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:23,968 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T14:15:23,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:23,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:23,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:23,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:23,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:23,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:23,969 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#C#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:23,970 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/71f9431b1b97495a8c25f89dd34cc6fa is 50, key is test_row_0/C:col10/1731680122935/Put/seqid=0 2024-11-15T14:15:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:23,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T14:15:23,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742025_1201 (size=31903) 2024-11-15T14:15:24,003 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/0ca06830d6904d988b27e0e6a0ec2d20 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0ca06830d6904d988b27e0e6a0ec2d20 2024-11-15T14:15:24,011 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/A of 1fd20151ae970369f5abd5f0c2e6e70d into 0ca06830d6904d988b27e0e6a0ec2d20(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:24,011 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:24,011 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/A, priority=13, startTime=1731680123863; duration=0sec 2024-11-15T14:15:24,012 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:24,012 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:A 2024-11-15T14:15:24,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742026_1202 (size=12949) 2024-11-15T14:15:24,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111523244042c39e43c39bc68f124ac30c6e_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680123621/Put/seqid=0 2024-11-15T14:15:24,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680184042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:24,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680184043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:24,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742027_1203 (size=12454) 2024-11-15T14:15:24,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680184162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:24,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680184163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T14:15:24,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680184373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:24,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680184373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:24,439 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/71f9431b1b97495a8c25f89dd34cc6fa as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/71f9431b1b97495a8c25f89dd34cc6fa 2024-11-15T14:15:24,455 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/C of 1fd20151ae970369f5abd5f0c2e6e70d into 71f9431b1b97495a8c25f89dd34cc6fa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:24,455 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:24,456 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/C, priority=13, startTime=1731680123863; duration=0sec 2024-11-15T14:15:24,456 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:24,456 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:C 2024-11-15T14:15:24,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:24,490 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111523244042c39e43c39bc68f124ac30c6e_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111523244042c39e43c39bc68f124ac30c6e_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:24,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/7c177f5b09e44c5fb5c762c8eb1b8090, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:24,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/7c177f5b09e44c5fb5c762c8eb1b8090 is 175, key is test_row_0/A:col10/1731680123621/Put/seqid=0 2024-11-15T14:15:24,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742028_1204 (size=31255) 2024-11-15T14:15:24,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680184692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680184692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T14:15:24,927 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/7c177f5b09e44c5fb5c762c8eb1b8090 2024-11-15T14:15:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/a2e5fa7cb37a4127940b7837c43536b3 is 50, key is test_row_0/B:col10/1731680123621/Put/seqid=0 2024-11-15T14:15:25,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742029_1205 (size=12301) 2024-11-15T14:15:25,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680185202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:25,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:25,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:25,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680185202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:25,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:25,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:25,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:25,403 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/a2e5fa7cb37a4127940b7837c43536b3 2024-11-15T14:15:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/8948120c4801469b850d668afc532628 is 50, key is test_row_0/C:col10/1731680123621/Put/seqid=0 2024-11-15T14:15:25,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742030_1206 (size=12301) 2024-11-15T14:15:25,466 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/8948120c4801469b850d668afc532628 2024-11-15T14:15:25,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/7c177f5b09e44c5fb5c762c8eb1b8090 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7c177f5b09e44c5fb5c762c8eb1b8090 2024-11-15T14:15:25,483 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7c177f5b09e44c5fb5c762c8eb1b8090, entries=150, sequenceid=394, filesize=30.5 K 2024-11-15T14:15:25,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/a2e5fa7cb37a4127940b7837c43536b3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a2e5fa7cb37a4127940b7837c43536b3 2024-11-15T14:15:25,493 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a2e5fa7cb37a4127940b7837c43536b3, entries=150, sequenceid=394, filesize=12.0 K 2024-11-15T14:15:25,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/8948120c4801469b850d668afc532628 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/8948120c4801469b850d668afc532628 2024-11-15T14:15:25,506 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/8948120c4801469b850d668afc532628, entries=150, sequenceid=394, filesize=12.0 K 2024-11-15T14:15:25,509 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1fd20151ae970369f5abd5f0c2e6e70d in 1541ms, sequenceid=394, compaction requested=false 2024-11-15T14:15:25,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:25,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:25,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-15T14:15:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-15T14:15:25,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-15T14:15:25,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8590 sec 2024-11-15T14:15:25,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8690 sec 2024-11-15T14:15:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T14:15:25,802 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:25,803 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-15T14:15:25,805 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T14:15:25,807 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:25,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T14:15:25,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T14:15:25,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:25,961 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T14:15:25,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:25,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:25,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:25,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:25,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159e78461e184342aaad4c89e4de0fc2e8_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680124038/Put/seqid=0 2024-11-15T14:15:26,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742031_1207 (size=12454) 2024-11-15T14:15:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T14:15:26,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:26,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680186274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:26,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680186274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:26,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680186392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:26,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680186392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:26,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:26,423 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159e78461e184342aaad4c89e4de0fc2e8_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159e78461e184342aaad4c89e4de0fc2e8_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:26,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/2b8e157781c342f29ac9709af279a675, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:26,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/2b8e157781c342f29ac9709af279a675 is 175, key is test_row_0/A:col10/1731680124038/Put/seqid=0 2024-11-15T14:15:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T14:15:26,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742032_1208 (size=31255) 2024-11-15T14:15:26,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680186602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:26,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680186602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:26,864 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/2b8e157781c342f29ac9709af279a675 2024-11-15T14:15:26,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/f12d3c1477124c18bd9908fcafa9d19c is 50, key is test_row_0/B:col10/1731680124038/Put/seqid=0 2024-11-15T14:15:26,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680186932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680186933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T14:15:26,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742033_1209 (size=12301) 2024-11-15T14:15:27,354 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/f12d3c1477124c18bd9908fcafa9d19c 2024-11-15T14:15:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/8346d7d9e7c54d869250e5d9d388c945 is 50, key is test_row_0/C:col10/1731680124038/Put/seqid=0 2024-11-15T14:15:27,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742034_1210 (size=12301) 2024-11-15T14:15:27,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680187442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:27,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680187452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:27,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:27,766 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/8346d7d9e7c54d869250e5d9d388c945 2024-11-15T14:15:27,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/2b8e157781c342f29ac9709af279a675 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2b8e157781c342f29ac9709af279a675 2024-11-15T14:15:27,776 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2b8e157781c342f29ac9709af279a675, entries=150, sequenceid=411, filesize=30.5 K 2024-11-15T14:15:27,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/f12d3c1477124c18bd9908fcafa9d19c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f12d3c1477124c18bd9908fcafa9d19c 2024-11-15T14:15:27,781 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f12d3c1477124c18bd9908fcafa9d19c, entries=150, sequenceid=411, filesize=12.0 K 2024-11-15T14:15:27,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/8346d7d9e7c54d869250e5d9d388c945 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/8346d7d9e7c54d869250e5d9d388c945 2024-11-15T14:15:27,788 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/8346d7d9e7c54d869250e5d9d388c945, entries=150, sequenceid=411, filesize=12.0 K 2024-11-15T14:15:27,789 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1fd20151ae970369f5abd5f0c2e6e70d in 1828ms, sequenceid=411, compaction requested=true 2024-11-15T14:15:27,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:27,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:27,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-15T14:15:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-15T14:15:27,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-15T14:15:27,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9830 sec 2024-11-15T14:15:27,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.9890 sec 2024-11-15T14:15:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T14:15:27,952 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:27,952 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-15T14:15:27,954 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T14:15:27,955 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:27,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T14:15:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:28,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T14:15:28,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:28,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:28,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:28,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:28,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:28,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:28,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a4f6a71ba1ab48a5b2f89008d51d418f_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680126218/Put/seqid=0 2024-11-15T14:15:28,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T14:15:28,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:28,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:28,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:28,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742035_1211 (size=14994) 2024-11-15T14:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,109 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:28,116 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a4f6a71ba1ab48a5b2f89008d51d418f_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a4f6a71ba1ab48a5b2f89008d51d418f_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:28,117 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/41d50988059c44c58a76373ac9dd5bae, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:28,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/41d50988059c44c58a76373ac9dd5bae is 175, key is test_row_0/A:col10/1731680126218/Put/seqid=0 2024-11-15T14:15:28,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731680188117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742036_1212 (size=39949) 2024-11-15T14:15:28,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731680188123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33914 deadline: 1731680188125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731680188232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731680188232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33914 deadline: 1731680188232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T14:15:28,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:28,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:28,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:28,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T14:15:28,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T14:15:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:28,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731680188442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731680188442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33914 deadline: 1731680188442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33862 deadline: 1731680188451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,454 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in 1fd20151ae970369f5abd5f0c2e6e70d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:28,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33912 deadline: 1731680188461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,463 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in 1fd20151ae970369f5abd5f0c2e6e70d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:28,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,529 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=429, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/41d50988059c44c58a76373ac9dd5bae 2024-11-15T14:15:28,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/8fcf7a3730224345a8e638bbadae8fd1 is 50, key is test_row_0/B:col10/1731680126218/Put/seqid=0 2024-11-15T14:15:28,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742037_1213 (size=12301) 2024-11-15T14:15:28,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T14:15:28,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:28,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:28,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:28,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T14:15:28,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T14:15:28,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:28,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:28,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:28,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731680188752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731680188752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33914 deadline: 1731680188752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:28,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:28,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:28,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T14:15:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:28,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:28,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/8fcf7a3730224345a8e638bbadae8fd1 2024-11-15T14:15:28,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/6c2e9f0692b943c997e4f3843618fab3 is 50, key is test_row_0/C:col10/1731680126218/Put/seqid=0 2024-11-15T14:15:28,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742038_1214 (size=12301) 2024-11-15T14:15:29,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T14:15:29,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:29,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:29,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:29,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:29,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T14:15:29,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T14:15:29,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:29,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:29,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:29,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:29,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:29,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33926 deadline: 1731680189262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:29,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33878 deadline: 1731680189262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:29,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33914 deadline: 1731680189262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:15:29,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T14:15:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:29,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:29,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:29,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/6c2e9f0692b943c997e4f3843618fab3 2024-11-15T14:15:29,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/41d50988059c44c58a76373ac9dd5bae as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/41d50988059c44c58a76373ac9dd5bae 2024-11-15T14:15:29,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/41d50988059c44c58a76373ac9dd5bae, entries=200, sequenceid=429, filesize=39.0 K 2024-11-15T14:15:29,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/8fcf7a3730224345a8e638bbadae8fd1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/8fcf7a3730224345a8e638bbadae8fd1 2024-11-15T14:15:29,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/8fcf7a3730224345a8e638bbadae8fd1, entries=150, sequenceid=429, filesize=12.0 K 2024-11-15T14:15:29,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/6c2e9f0692b943c997e4f3843618fab3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/6c2e9f0692b943c997e4f3843618fab3 2024-11-15T14:15:29,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/6c2e9f0692b943c997e4f3843618fab3, entries=150, sequenceid=429, filesize=12.0 K 2024-11-15T14:15:29,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1fd20151ae970369f5abd5f0c2e6e70d in 1285ms, sequenceid=429, compaction requested=true 2024-11-15T14:15:29,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:29,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:29,379 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:29,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:29,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:29,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:29,379 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:29,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fd20151ae970369f5abd5f0c2e6e70d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:29,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:29,381 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:29,381 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:29,381 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/B is initiating minor compaction (all files) 2024-11-15T14:15:29,381 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/A is initiating minor compaction (all files) 2024-11-15T14:15:29,381 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/A in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:29,381 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/B in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:29,382 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0ca06830d6904d988b27e0e6a0ec2d20, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7c177f5b09e44c5fb5c762c8eb1b8090, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2b8e157781c342f29ac9709af279a675, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/41d50988059c44c58a76373ac9dd5bae] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=131.2 K 2024-11-15T14:15:29,382 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/19feb7a465ba4f83a155ba8e99aa1b3a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a2e5fa7cb37a4127940b7837c43536b3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f12d3c1477124c18bd9908fcafa9d19c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/8fcf7a3730224345a8e638bbadae8fd1] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=48.7 K 2024-11-15T14:15:29,382 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:29,382 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0ca06830d6904d988b27e0e6a0ec2d20, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7c177f5b09e44c5fb5c762c8eb1b8090, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2b8e157781c342f29ac9709af279a675, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/41d50988059c44c58a76373ac9dd5bae] 2024-11-15T14:15:29,382 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ca06830d6904d988b27e0e6a0ec2d20, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731680122923 2024-11-15T14:15:29,382 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 19feb7a465ba4f83a155ba8e99aa1b3a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731680122923 2024-11-15T14:15:29,382 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c177f5b09e44c5fb5c762c8eb1b8090, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731680123621 2024-11-15T14:15:29,383 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e5fa7cb37a4127940b7837c43536b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731680123621 2024-11-15T14:15:29,383 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b8e157781c342f29ac9709af279a675, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731680124029 2024-11-15T14:15:29,383 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting f12d3c1477124c18bd9908fcafa9d19c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731680124029 2024-11-15T14:15:29,383 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41d50988059c44c58a76373ac9dd5bae, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731680126218 2024-11-15T14:15:29,383 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fcf7a3730224345a8e638bbadae8fd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731680126218 2024-11-15T14:15:29,390 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:29,392 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#B#compaction#176 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:29,392 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115f17a36f18f2647da9887581cd6e03dd9_1fd20151ae970369f5abd5f0c2e6e70d store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:29,392 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/818c966745684723bab809f56c500988 is 50, key is test_row_0/B:col10/1731680126218/Put/seqid=0 2024-11-15T14:15:29,394 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115f17a36f18f2647da9887581cd6e03dd9_1fd20151ae970369f5abd5f0c2e6e70d, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:29,395 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f17a36f18f2647da9887581cd6e03dd9_1fd20151ae970369f5abd5f0c2e6e70d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:29,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742039_1215 (size=13085) 2024-11-15T14:15:29,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742040_1216 (size=4469) 2024-11-15T14:15:29,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T14:15:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:29,489 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T14:15:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:29,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:29,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f6437263f2784a5ba90a03c7d1c9d9db_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680128114/Put/seqid=0 2024-11-15T14:15:29,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742041_1217 (size=12454) 2024-11-15T14:15:29,621 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-15T14:15:29,621 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:29,621 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:29,621 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:29,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:29,622 INFO [Thread-631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-631. 2024-11-15T14:15:29,622 DEBUG [Thread-631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:29,622 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:29,622 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:29,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:29,623 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-11-15T14:15:29,623 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:29,623 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:29,623 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:29,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:29,625 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-15T14:15:29,625 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:29,625 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:29,625 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:29,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:29,803 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#A#compaction#177 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:29,804 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/cbc37a8fa67c4737885a1ffd30177768 is 175, key is test_row_0/A:col10/1731680126218/Put/seqid=0 2024-11-15T14:15:29,809 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/818c966745684723bab809f56c500988 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/818c966745684723bab809f56c500988 2024-11-15T14:15:29,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742042_1218 (size=32039) 2024-11-15T14:15:29,814 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/B of 1fd20151ae970369f5abd5f0c2e6e70d into 818c966745684723bab809f56c500988(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:29,814 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:29,814 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/B, priority=12, startTime=1731680129379; duration=0sec 2024-11-15T14:15:29,814 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:29,814 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:B 2024-11-15T14:15:29,814 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:29,816 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:29,816 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): 1fd20151ae970369f5abd5f0c2e6e70d/C is initiating minor compaction (all files) 2024-11-15T14:15:29,816 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fd20151ae970369f5abd5f0c2e6e70d/C in TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:29,816 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/71f9431b1b97495a8c25f89dd34cc6fa, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/8948120c4801469b850d668afc532628, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/8346d7d9e7c54d869250e5d9d388c945, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/6c2e9f0692b943c997e4f3843618fab3] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp, totalSize=48.7 K 2024-11-15T14:15:29,816 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 71f9431b1b97495a8c25f89dd34cc6fa, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731680122923 2024-11-15T14:15:29,816 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 8948120c4801469b850d668afc532628, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731680123621 2024-11-15T14:15:29,817 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 8346d7d9e7c54d869250e5d9d388c945, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731680124029 2024-11-15T14:15:29,817 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c2e9f0692b943c997e4f3843618fab3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731680126218 2024-11-15T14:15:29,824 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fd20151ae970369f5abd5f0c2e6e70d#C#compaction#179 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:29,824 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/1f57083d676043128d59a69d2d54329d is 50, key is test_row_0/C:col10/1731680126218/Put/seqid=0 2024-11-15T14:15:29,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742043_1219 (size=13085) 2024-11-15T14:15:29,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:29,908 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f6437263f2784a5ba90a03c7d1c9d9db_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f6437263f2784a5ba90a03c7d1c9d9db_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:29,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/b2ab3d3b4bef47d1a53d49dc8f047eb8, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:29,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/b2ab3d3b4bef47d1a53d49dc8f047eb8 is 175, key is test_row_0/A:col10/1731680128114/Put/seqid=0 2024-11-15T14:15:29,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742044_1220 (size=31255) 2024-11-15T14:15:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T14:15:30,222 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/cbc37a8fa67c4737885a1ffd30177768 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/cbc37a8fa67c4737885a1ffd30177768 2024-11-15T14:15:30,229 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/A of 1fd20151ae970369f5abd5f0c2e6e70d into cbc37a8fa67c4737885a1ffd30177768(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:30,229 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:30,229 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/A, priority=12, startTime=1731680129379; duration=0sec 2024-11-15T14:15:30,229 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:30,230 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:A 2024-11-15T14:15:30,234 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/1f57083d676043128d59a69d2d54329d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1f57083d676043128d59a69d2d54329d 2024-11-15T14:15:30,239 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fd20151ae970369f5abd5f0c2e6e70d/C of 1fd20151ae970369f5abd5f0c2e6e70d into 1f57083d676043128d59a69d2d54329d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:30,239 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:30,240 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., storeName=1fd20151ae970369f5abd5f0c2e6e70d/C, priority=12, startTime=1731680129379; duration=0sec 2024-11-15T14:15:30,240 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:30,240 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fd20151ae970369f5abd5f0c2e6e70d:C 2024-11-15T14:15:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:30,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. as already flushing 2024-11-15T14:15:30,284 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-15T14:15:30,284 INFO [Thread-620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-620. 2024-11-15T14:15:30,284 DEBUG [Thread-620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:30,284 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:30,284 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:30,284 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:30,284 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:30,284 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:30,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:30,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:30,284 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-15T14:15:30,284 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:30,284 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:30,284 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:30,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:30,314 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=443, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/b2ab3d3b4bef47d1a53d49dc8f047eb8 2024-11-15T14:15:30,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/2260ca590481491eaafa647521450926 is 50, key is test_row_0/B:col10/1731680128114/Put/seqid=0 2024-11-15T14:15:30,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742045_1221 (size=12301) 2024-11-15T14:15:30,476 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-15T14:15:30,476 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:30,477 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:30,477 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:30,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:30,485 INFO [Thread-618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-618. 2024-11-15T14:15:30,485 DEBUG [Thread-618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:30,485 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:30,486 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:30,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:30,726 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/2260ca590481491eaafa647521450926 2024-11-15T14:15:30,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/f66f6fcde98a429780c709b5244e041b is 50, key is test_row_0/C:col10/1731680128114/Put/seqid=0 2024-11-15T14:15:30,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742046_1222 (size=12301) 2024-11-15T14:15:31,138 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/f66f6fcde98a429780c709b5244e041b 2024-11-15T14:15:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/b2ab3d3b4bef47d1a53d49dc8f047eb8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/b2ab3d3b4bef47d1a53d49dc8f047eb8 2024-11-15T14:15:31,154 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/b2ab3d3b4bef47d1a53d49dc8f047eb8, entries=150, sequenceid=443, filesize=30.5 K 2024-11-15T14:15:31,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/2260ca590481491eaafa647521450926 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/2260ca590481491eaafa647521450926 2024-11-15T14:15:31,160 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/2260ca590481491eaafa647521450926, entries=150, sequenceid=443, filesize=12.0 K 2024-11-15T14:15:31,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/f66f6fcde98a429780c709b5244e041b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/f66f6fcde98a429780c709b5244e041b 2024-11-15T14:15:31,165 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/f66f6fcde98a429780c709b5244e041b, entries=150, sequenceid=443, filesize=12.0 K 2024-11-15T14:15:31,165 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=33.54 KB/34350 for 1fd20151ae970369f5abd5f0c2e6e70d in 1676ms, sequenceid=443, compaction requested=false 2024-11-15T14:15:31,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 1fd20151ae970369f5abd5f0c2e6e70d: 2024-11-15T14:15:31,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:31,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-15T14:15:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-15T14:15:31,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-15T14:15:31,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2120 sec 2024-11-15T14:15:31,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 3.2160 sec 2024-11-15T14:15:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T14:15:32,122 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 165 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 164 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2063 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2017 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 901 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2703 rows 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 900 2024-11-15T14:15:32,123 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2694 rows 2024-11-15T14:15:32,123 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T14:15:32,123 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T14:15:32,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:32,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:32,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:32,124 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T14:15:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T14:15:32,127 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680132127"}]},"ts":"1731680132127"} 2024-11-15T14:15:32,129 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T14:15:32,129 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T14:15:32,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T14:15:32,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, UNASSIGN}] 2024-11-15T14:15:32,131 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, UNASSIGN 2024-11-15T14:15:32,132 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1fd20151ae970369f5abd5f0c2e6e70d, regionState=CLOSING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:15:32,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, UNASSIGN because future has completed 2024-11-15T14:15:32,134 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T14:15:32,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:15:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T14:15:32,287 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:32,287 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T14:15:32,287 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 1fd20151ae970369f5abd5f0c2e6e70d, disabling compactions & flushes 2024-11-15T14:15:32,287 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:32,287 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:32,288 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. after waiting 0 ms 2024-11-15T14:15:32,288 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:32,288 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 1fd20151ae970369f5abd5f0c2e6e70d 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T14:15:32,288 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=A 2024-11-15T14:15:32,288 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:32,288 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=B 2024-11-15T14:15:32,288 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:32,289 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fd20151ae970369f5abd5f0c2e6e70d, store=C 2024-11-15T14:15:32,289 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:32,299 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111539b44482e5904fc6aa0b3f5058c39447_1fd20151ae970369f5abd5f0c2e6e70d is 50, key is test_row_0/A:col10/1731680130483/Put/seqid=0 2024-11-15T14:15:32,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742047_1223 (size=12454) 2024-11-15T14:15:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T14:15:32,704 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:32,709 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111539b44482e5904fc6aa0b3f5058c39447_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111539b44482e5904fc6aa0b3f5058c39447_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:32,711 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/536b82121e0a4e3dbd4ac1177b9b265d, store: [table=TestAcidGuarantees family=A region=1fd20151ae970369f5abd5f0c2e6e70d] 2024-11-15T14:15:32,711 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/536b82121e0a4e3dbd4ac1177b9b265d is 175, key is test_row_0/A:col10/1731680130483/Put/seqid=0 2024-11-15T14:15:32,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742048_1224 (size=31255) 2024-11-15T14:15:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T14:15:33,117 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=454, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/536b82121e0a4e3dbd4ac1177b9b265d 2024-11-15T14:15:33,130 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/e37e9b240c0f4c35bf04b6132c758b29 is 50, key is test_row_0/B:col10/1731680130483/Put/seqid=0 2024-11-15T14:15:33,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742049_1225 (size=12301) 2024-11-15T14:15:33,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T14:15:33,536 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/e37e9b240c0f4c35bf04b6132c758b29 2024-11-15T14:15:33,548 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/e8f6b6b3e83746e686399d9c20d0dcd8 is 50, key is test_row_0/C:col10/1731680130483/Put/seqid=0 2024-11-15T14:15:33,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742050_1226 (size=12301) 2024-11-15T14:15:33,953 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/e8f6b6b3e83746e686399d9c20d0dcd8 2024-11-15T14:15:33,963 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/A/536b82121e0a4e3dbd4ac1177b9b265d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/536b82121e0a4e3dbd4ac1177b9b265d 2024-11-15T14:15:33,967 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/536b82121e0a4e3dbd4ac1177b9b265d, entries=150, sequenceid=454, filesize=30.5 K 2024-11-15T14:15:33,968 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/B/e37e9b240c0f4c35bf04b6132c758b29 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/e37e9b240c0f4c35bf04b6132c758b29 2024-11-15T14:15:33,973 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/e37e9b240c0f4c35bf04b6132c758b29, entries=150, sequenceid=454, filesize=12.0 K 2024-11-15T14:15:33,973 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/.tmp/C/e8f6b6b3e83746e686399d9c20d0dcd8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/e8f6b6b3e83746e686399d9c20d0dcd8 2024-11-15T14:15:33,978 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/e8f6b6b3e83746e686399d9c20d0dcd8, entries=150, sequenceid=454, filesize=12.0 K 2024-11-15T14:15:33,979 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1fd20151ae970369f5abd5f0c2e6e70d in 1690ms, sequenceid=454, compaction requested=true 2024-11-15T14:15:33,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee812d33acde4d6dab7bc447c2fa9b73, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee1d53373c4e4a7aaa49f2b09346ce9c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/18acfa812f1642598a5f669be1d27f03, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/5bcf2a2ca5f140998e10f7184f487c17, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7047870c23cd49808bcd8e9044f6106b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4181c72c2d78450f83cb2a328632c4d9, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0b9cdbe1342040e48ca5b03d040472b1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/fc3f178d02094dc0ae728ffb4b79e504, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/3cbfe2fd14ab4939aa26ba845624e36f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/30faf6e2a6dc456c90ced667cd5561ca, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/d413db03409947be813441ca2418f685, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2cfdf6bcff0c4c9891a1ac8d23a9096c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0064f858fcb24b3b80e0c2f6627ea137, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/893801de8a6b456792d7127ffa079dd3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/e2712b3287d04eaea8ec57c8be475d41, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/366b62c6698246bd80526a606b812e18, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2325c8db66e544d5abab5b7ae96ba2e5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4541c3f2254a493497de5e454902c27d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/cc5097831abb451b8e933dfd902d6ddc, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0ca06830d6904d988b27e0e6a0ec2d20, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7c177f5b09e44c5fb5c762c8eb1b8090, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2b8e157781c342f29ac9709af279a675, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/41d50988059c44c58a76373ac9dd5bae] to archive 2024-11-15T14:15:33,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:15:33,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee812d33acde4d6dab7bc447c2fa9b73 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee812d33acde4d6dab7bc447c2fa9b73 2024-11-15T14:15:33,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee1d53373c4e4a7aaa49f2b09346ce9c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/ee1d53373c4e4a7aaa49f2b09346ce9c 2024-11-15T14:15:33,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/18acfa812f1642598a5f669be1d27f03 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/18acfa812f1642598a5f669be1d27f03 2024-11-15T14:15:33,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/5bcf2a2ca5f140998e10f7184f487c17 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/5bcf2a2ca5f140998e10f7184f487c17 2024-11-15T14:15:33,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7047870c23cd49808bcd8e9044f6106b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7047870c23cd49808bcd8e9044f6106b 2024-11-15T14:15:33,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4181c72c2d78450f83cb2a328632c4d9 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4181c72c2d78450f83cb2a328632c4d9 2024-11-15T14:15:33,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0b9cdbe1342040e48ca5b03d040472b1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0b9cdbe1342040e48ca5b03d040472b1 2024-11-15T14:15:33,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/fc3f178d02094dc0ae728ffb4b79e504 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/fc3f178d02094dc0ae728ffb4b79e504 2024-11-15T14:15:33,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/3cbfe2fd14ab4939aa26ba845624e36f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/3cbfe2fd14ab4939aa26ba845624e36f 2024-11-15T14:15:33,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/30faf6e2a6dc456c90ced667cd5561ca to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/30faf6e2a6dc456c90ced667cd5561ca 2024-11-15T14:15:33,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/d413db03409947be813441ca2418f685 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/d413db03409947be813441ca2418f685 2024-11-15T14:15:33,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2cfdf6bcff0c4c9891a1ac8d23a9096c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2cfdf6bcff0c4c9891a1ac8d23a9096c 2024-11-15T14:15:33,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0064f858fcb24b3b80e0c2f6627ea137 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0064f858fcb24b3b80e0c2f6627ea137 2024-11-15T14:15:33,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/893801de8a6b456792d7127ffa079dd3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/893801de8a6b456792d7127ffa079dd3 2024-11-15T14:15:33,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/e2712b3287d04eaea8ec57c8be475d41 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/e2712b3287d04eaea8ec57c8be475d41 2024-11-15T14:15:33,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/366b62c6698246bd80526a606b812e18 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/366b62c6698246bd80526a606b812e18 2024-11-15T14:15:33,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2325c8db66e544d5abab5b7ae96ba2e5 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2325c8db66e544d5abab5b7ae96ba2e5 2024-11-15T14:15:34,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4541c3f2254a493497de5e454902c27d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/4541c3f2254a493497de5e454902c27d 2024-11-15T14:15:34,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/cc5097831abb451b8e933dfd902d6ddc to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/cc5097831abb451b8e933dfd902d6ddc 2024-11-15T14:15:34,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0ca06830d6904d988b27e0e6a0ec2d20 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/0ca06830d6904d988b27e0e6a0ec2d20 2024-11-15T14:15:34,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7c177f5b09e44c5fb5c762c8eb1b8090 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/7c177f5b09e44c5fb5c762c8eb1b8090 2024-11-15T14:15:34,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2b8e157781c342f29ac9709af279a675 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/2b8e157781c342f29ac9709af279a675 2024-11-15T14:15:34,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/41d50988059c44c58a76373ac9dd5bae to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/41d50988059c44c58a76373ac9dd5bae 2024-11-15T14:15:34,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/1b4a69612326482faec3e1003fdcdbfa, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/ce987213503b482298811f328e488bc1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9babfcf492474a5db90f2efe43906740, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9a150f956d9e4b3cbfef772573b19ebf, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a55fa84b26f240978c8cb2eaf4c5bcf4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f239d11ccb154802ae608b5b4b18098b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/87384dda24c342fabfc8199beff79ecf, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/c45b9d77ffb14c7b816281f5ecdfd0eb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/cc4ea404733049168a4c0e9666c0067a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f71d1074930d4f79a3ecdbf91890c913, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/75a25df2f70e4acba70b82ed160d9f63, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/6e21dd28a53d49ca9c68f2d777fc4e71, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/460487d1926640cea3c413aec36604f2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9b10832fbbae47cb98fc6825629aaa02, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/828584426f64424d9433df2357e3111a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f7e3bfd1d3d54b61bc3471575d4151c6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/5ad459c938814ac2920cf8db8e41f974, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a0ff0fa4d418442086ccb8391459fec7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/19feb7a465ba4f83a155ba8e99aa1b3a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/54958d95d0404a76b911880af1af9a00, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a2e5fa7cb37a4127940b7837c43536b3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f12d3c1477124c18bd9908fcafa9d19c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/8fcf7a3730224345a8e638bbadae8fd1] to archive 2024-11-15T14:15:34,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:15:34,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/1b4a69612326482faec3e1003fdcdbfa to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/1b4a69612326482faec3e1003fdcdbfa 2024-11-15T14:15:34,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/ce987213503b482298811f328e488bc1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/ce987213503b482298811f328e488bc1 2024-11-15T14:15:34,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9babfcf492474a5db90f2efe43906740 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9babfcf492474a5db90f2efe43906740 2024-11-15T14:15:34,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9a150f956d9e4b3cbfef772573b19ebf to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9a150f956d9e4b3cbfef772573b19ebf 2024-11-15T14:15:34,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a55fa84b26f240978c8cb2eaf4c5bcf4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a55fa84b26f240978c8cb2eaf4c5bcf4 2024-11-15T14:15:34,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f239d11ccb154802ae608b5b4b18098b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f239d11ccb154802ae608b5b4b18098b 2024-11-15T14:15:34,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/87384dda24c342fabfc8199beff79ecf to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/87384dda24c342fabfc8199beff79ecf 2024-11-15T14:15:34,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/c45b9d77ffb14c7b816281f5ecdfd0eb to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/c45b9d77ffb14c7b816281f5ecdfd0eb 2024-11-15T14:15:34,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/cc4ea404733049168a4c0e9666c0067a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/cc4ea404733049168a4c0e9666c0067a 2024-11-15T14:15:34,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f71d1074930d4f79a3ecdbf91890c913 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f71d1074930d4f79a3ecdbf91890c913 2024-11-15T14:15:34,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/75a25df2f70e4acba70b82ed160d9f63 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/75a25df2f70e4acba70b82ed160d9f63 2024-11-15T14:15:34,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/6e21dd28a53d49ca9c68f2d777fc4e71 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/6e21dd28a53d49ca9c68f2d777fc4e71 2024-11-15T14:15:34,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/460487d1926640cea3c413aec36604f2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/460487d1926640cea3c413aec36604f2 2024-11-15T14:15:34,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9b10832fbbae47cb98fc6825629aaa02 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/9b10832fbbae47cb98fc6825629aaa02 2024-11-15T14:15:34,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/828584426f64424d9433df2357e3111a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/828584426f64424d9433df2357e3111a 2024-11-15T14:15:34,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f7e3bfd1d3d54b61bc3471575d4151c6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f7e3bfd1d3d54b61bc3471575d4151c6 2024-11-15T14:15:34,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/5ad459c938814ac2920cf8db8e41f974 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/5ad459c938814ac2920cf8db8e41f974 2024-11-15T14:15:34,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a0ff0fa4d418442086ccb8391459fec7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a0ff0fa4d418442086ccb8391459fec7 2024-11-15T14:15:34,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/19feb7a465ba4f83a155ba8e99aa1b3a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/19feb7a465ba4f83a155ba8e99aa1b3a 2024-11-15T14:15:34,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/54958d95d0404a76b911880af1af9a00 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/54958d95d0404a76b911880af1af9a00 2024-11-15T14:15:34,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a2e5fa7cb37a4127940b7837c43536b3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/a2e5fa7cb37a4127940b7837c43536b3 2024-11-15T14:15:34,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f12d3c1477124c18bd9908fcafa9d19c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/f12d3c1477124c18bd9908fcafa9d19c 2024-11-15T14:15:34,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/8fcf7a3730224345a8e638bbadae8fd1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/8fcf7a3730224345a8e638bbadae8fd1 2024-11-15T14:15:34,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/cc2105cc41474ab1b4305245e8e0dc5b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/0794f4b2956a4bb0995447ba470cf361, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/106a6752f6224de2b0dbcfe9e66aa157, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/53d39863c3b241a5bdec6f5a377312b6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/dce8cadfd87b460cb8080027746187c2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1fc651ca16a84da19172d6d6dc993389, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/7dcd4d1ba9634da89964639c8261f62e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/42adeb719b0346a4ae36cc365927e12a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/ce4d13d2ed504144b818794c1cdfda8f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/e3e991d5e583431787ba1eecd3c6c460, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/0a67082a13b045cb86df942d8c0b93d3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/a208f2e625c046199d95d7b151f9fac6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/9c810afd43e841c38cf1f049172e177b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/9765a483bd314b0292bb6d933e4b52be, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/585c472ae5964df1b72ae3330927a78d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/844c4668a1504cf3991eac56d736e821, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/7b745ea9ca284da1940eea7c83938797, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1775728b58974f3996ed9c2d8b001776, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/71f9431b1b97495a8c25f89dd34cc6fa, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/5585a0ef670d435c8b089f0497458c33, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/8948120c4801469b850d668afc532628, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/8346d7d9e7c54d869250e5d9d388c945, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/6c2e9f0692b943c997e4f3843618fab3] to archive 2024-11-15T14:15:34,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:15:34,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/cc2105cc41474ab1b4305245e8e0dc5b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/cc2105cc41474ab1b4305245e8e0dc5b 2024-11-15T14:15:34,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/0794f4b2956a4bb0995447ba470cf361 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/0794f4b2956a4bb0995447ba470cf361 2024-11-15T14:15:34,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/106a6752f6224de2b0dbcfe9e66aa157 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/106a6752f6224de2b0dbcfe9e66aa157 2024-11-15T14:15:34,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/53d39863c3b241a5bdec6f5a377312b6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/53d39863c3b241a5bdec6f5a377312b6 2024-11-15T14:15:34,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/dce8cadfd87b460cb8080027746187c2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/dce8cadfd87b460cb8080027746187c2 2024-11-15T14:15:34,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1fc651ca16a84da19172d6d6dc993389 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1fc651ca16a84da19172d6d6dc993389 2024-11-15T14:15:34,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/7dcd4d1ba9634da89964639c8261f62e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/7dcd4d1ba9634da89964639c8261f62e 2024-11-15T14:15:34,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/42adeb719b0346a4ae36cc365927e12a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/42adeb719b0346a4ae36cc365927e12a 2024-11-15T14:15:34,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/ce4d13d2ed504144b818794c1cdfda8f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/ce4d13d2ed504144b818794c1cdfda8f 2024-11-15T14:15:34,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/e3e991d5e583431787ba1eecd3c6c460 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/e3e991d5e583431787ba1eecd3c6c460 2024-11-15T14:15:34,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/0a67082a13b045cb86df942d8c0b93d3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/0a67082a13b045cb86df942d8c0b93d3 2024-11-15T14:15:34,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/a208f2e625c046199d95d7b151f9fac6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/a208f2e625c046199d95d7b151f9fac6 2024-11-15T14:15:34,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/9c810afd43e841c38cf1f049172e177b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/9c810afd43e841c38cf1f049172e177b 2024-11-15T14:15:34,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/9765a483bd314b0292bb6d933e4b52be to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/9765a483bd314b0292bb6d933e4b52be 2024-11-15T14:15:34,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/585c472ae5964df1b72ae3330927a78d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/585c472ae5964df1b72ae3330927a78d 2024-11-15T14:15:34,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/844c4668a1504cf3991eac56d736e821 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/844c4668a1504cf3991eac56d736e821 2024-11-15T14:15:34,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/7b745ea9ca284da1940eea7c83938797 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/7b745ea9ca284da1940eea7c83938797 2024-11-15T14:15:34,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1775728b58974f3996ed9c2d8b001776 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1775728b58974f3996ed9c2d8b001776 2024-11-15T14:15:34,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/71f9431b1b97495a8c25f89dd34cc6fa to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/71f9431b1b97495a8c25f89dd34cc6fa 2024-11-15T14:15:34,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/5585a0ef670d435c8b089f0497458c33 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/5585a0ef670d435c8b089f0497458c33 2024-11-15T14:15:34,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/8948120c4801469b850d668afc532628 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/8948120c4801469b850d668afc532628 2024-11-15T14:15:34,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/8346d7d9e7c54d869250e5d9d388c945 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/8346d7d9e7c54d869250e5d9d388c945 2024-11-15T14:15:34,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/6c2e9f0692b943c997e4f3843618fab3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/6c2e9f0692b943c997e4f3843618fab3 2024-11-15T14:15:34,076 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/recovered.edits/457.seqid, newMaxSeqId=457, maxSeqId=4 2024-11-15T14:15:34,077 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. 2024-11-15T14:15:34,077 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 1fd20151ae970369f5abd5f0c2e6e70d: Waiting for close lock at 1731680132287Running coprocessor pre-close hooks at 1731680132287Disabling compacts and flushes for region at 1731680132287Disabling writes for close at 1731680132288 (+1 ms)Obtaining lock to block concurrent updates at 1731680132288Preparing flush snapshotting stores in 1fd20151ae970369f5abd5f0c2e6e70d at 1731680132288Finished memstore snapshotting TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731680132289 (+1 ms)Flushing stores of TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d. at 1731680132290 (+1 ms)Flushing 1fd20151ae970369f5abd5f0c2e6e70d/A: creating writer at 1731680132290Flushing mob file 1fd20151ae970369f5abd5f0c2e6e70d/A: appending metadata at 1731680132299 (+9 ms)Flushing mob file 1fd20151ae970369f5abd5f0c2e6e70d/A: closing flushed file at 1731680132299Flushing 1fd20151ae970369f5abd5f0c2e6e70d/A: appending metadata at 1731680132711 (+412 ms)Flushing 1fd20151ae970369f5abd5f0c2e6e70d/A: closing flushed file at 1731680132711Flushing 1fd20151ae970369f5abd5f0c2e6e70d/B: creating writer at 1731680133125 (+414 ms)Flushing 1fd20151ae970369f5abd5f0c2e6e70d/B: appending metadata at 1731680133130 (+5 ms)Flushing 1fd20151ae970369f5abd5f0c2e6e70d/B: closing flushed file at 1731680133130Flushing 1fd20151ae970369f5abd5f0c2e6e70d/C: creating writer at 1731680133544 (+414 ms)Flushing 1fd20151ae970369f5abd5f0c2e6e70d/C: appending metadata at 1731680133547 (+3 ms)Flushing 1fd20151ae970369f5abd5f0c2e6e70d/C: closing flushed file at 1731680133547Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18aa437d: reopening flushed file at 1731680133962 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@360953a0: reopening flushed file at 1731680133967 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2471c887: reopening flushed file at 1731680133973 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1fd20151ae970369f5abd5f0c2e6e70d in 1690ms, sequenceid=454, compaction requested=true at 1731680133979 (+6 ms)Writing region close event to WAL at 1731680134072 (+93 ms)Running coprocessor post-close hooks at 1731680134076 (+4 ms)Closed at 1731680134077 (+1 ms) 2024-11-15T14:15:34,079 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,080 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1fd20151ae970369f5abd5f0c2e6e70d, regionState=CLOSED 2024-11-15T14:15:34,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:15:34,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-15T14:15:34,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 1fd20151ae970369f5abd5f0c2e6e70d, server=26754e923c4a,40177,1731680077530 in 1.9480 sec 2024-11-15T14:15:34,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-15T14:15:34,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fd20151ae970369f5abd5f0c2e6e70d, UNASSIGN in 1.9530 sec 2024-11-15T14:15:34,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-15T14:15:34,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9570 sec 2024-11-15T14:15:34,089 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680134089"}]},"ts":"1731680134089"} 2024-11-15T14:15:34,091 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T14:15:34,091 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T14:15:34,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9670 sec 2024-11-15T14:15:34,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T14:15:34,282 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:34,283 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T14:15:34,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:34,287 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T14:15:34,288 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:34,293 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,297 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/recovered.edits] 2024-11-15T14:15:34,301 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/536b82121e0a4e3dbd4ac1177b9b265d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/536b82121e0a4e3dbd4ac1177b9b265d 2024-11-15T14:15:34,303 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/b2ab3d3b4bef47d1a53d49dc8f047eb8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/b2ab3d3b4bef47d1a53d49dc8f047eb8 2024-11-15T14:15:34,305 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/cbc37a8fa67c4737885a1ffd30177768 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/A/cbc37a8fa67c4737885a1ffd30177768 2024-11-15T14:15:34,308 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/2260ca590481491eaafa647521450926 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/2260ca590481491eaafa647521450926 2024-11-15T14:15:34,310 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/818c966745684723bab809f56c500988 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/818c966745684723bab809f56c500988 2024-11-15T14:15:34,311 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/e37e9b240c0f4c35bf04b6132c758b29 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/B/e37e9b240c0f4c35bf04b6132c758b29 2024-11-15T14:15:34,314 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1f57083d676043128d59a69d2d54329d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/1f57083d676043128d59a69d2d54329d 2024-11-15T14:15:34,315 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/e8f6b6b3e83746e686399d9c20d0dcd8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/e8f6b6b3e83746e686399d9c20d0dcd8 2024-11-15T14:15:34,316 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/f66f6fcde98a429780c709b5244e041b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/C/f66f6fcde98a429780c709b5244e041b 2024-11-15T14:15:34,320 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/recovered.edits/457.seqid to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d/recovered.edits/457.seqid 2024-11-15T14:15:34,321 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,321 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T14:15:34,321 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T14:15:34,322 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T14:15:34,329 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150926f411a2824d7f883978121b3decf8_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150926f411a2824d7f883978121b3decf8_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,330 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150fb79dc5987944e7805c834efdd10758_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150fb79dc5987944e7805c834efdd10758_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,332 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111523244042c39e43c39bc68f124ac30c6e_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111523244042c39e43c39bc68f124ac30c6e_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,333 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115271b049b5fb5462987b67305cb2bb7d3_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115271b049b5fb5462987b67305cb2bb7d3_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,334 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111539b44482e5904fc6aa0b3f5058c39447_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111539b44482e5904fc6aa0b3f5058c39447_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,336 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154ad96287d3d94ac4a8ee4890dbfa039f_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154ad96287d3d94ac4a8ee4890dbfa039f_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,337 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155490cd06c03f4773b81b2e23ca1222cd_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155490cd06c03f4773b81b2e23ca1222cd_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,339 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155f19d055d2e9428aa864155aed2ec6fc_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155f19d055d2e9428aa864155aed2ec6fc_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,340 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157ce9d266cfef4f37988c9d0c0fa90ad5_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157ce9d266cfef4f37988c9d0c0fa90ad5_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,341 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158055612172304d2ea9c470a2e0459194_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158055612172304d2ea9c470a2e0459194_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,342 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111584b4f0d625d84e20a48038f7a0f80ebe_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111584b4f0d625d84e20a48038f7a0f80ebe_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,343 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158b4b52362311492b93fd22acd4e568da_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158b4b52362311492b93fd22acd4e568da_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,344 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159e78461e184342aaad4c89e4de0fc2e8_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159e78461e184342aaad4c89e4de0fc2e8_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,345 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a4f6a71ba1ab48a5b2f89008d51d418f_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a4f6a71ba1ab48a5b2f89008d51d418f_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,345 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b2437f0e6335459ca616c86c34aba507_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b2437f0e6335459ca616c86c34aba507_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,346 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf1a4923820b4e6aa5e81e3444cffd3c_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf1a4923820b4e6aa5e81e3444cffd3c_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,347 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f6437263f2784a5ba90a03c7d1c9d9db_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f6437263f2784a5ba90a03c7d1c9d9db_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,348 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fa4cafdde1f7479bbd40c34ea8750fa3_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fa4cafdde1f7479bbd40c34ea8750fa3_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,349 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115faae6fbe1d2d443e950f4267ac607628_1fd20151ae970369f5abd5f0c2e6e70d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115faae6fbe1d2d443e950f4267ac607628_1fd20151ae970369f5abd5f0c2e6e70d 2024-11-15T14:15:34,350 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T14:15:34,352 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:34,355 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T14:15:34,357 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T14:15:34,358 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:34,358 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T14:15:34,358 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731680134358"}]},"ts":"9223372036854775807"} 2024-11-15T14:15:34,360 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T14:15:34,360 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1fd20151ae970369f5abd5f0c2e6e70d, NAME => 'TestAcidGuarantees,,1731680106748.1fd20151ae970369f5abd5f0c2e6e70d.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T14:15:34,360 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T14:15:34,360 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731680134360"}]},"ts":"9223372036854775807"} 2024-11-15T14:15:34,362 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T14:15:34,363 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:34,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-15T14:15:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T14:15:34,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T14:15:34,392 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:34,400 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=254 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1873743887_22 at /127.0.0.1:33242 [Waiting for operation #481] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1762377796_22 at /127.0.0.1:33226 [Waiting for operation #485] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/cluster_254b7fe7-4d7b-6185-a297-1a903e5d7b50/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/cluster_254b7fe7-4d7b-6185-a297-1a903e5d7b50/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1873743887_22 at /127.0.0.1:44146 [Waiting for operation #901] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1762377796_22 at /127.0.0.1:37588 [Waiting for operation #972] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 655) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=263 (was 265) 2024-11-15T14:15:34,409 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=254, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=263 2024-11-15T14:15:34,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T14:15:34,411 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T14:15:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T14:15:34,414 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T14:15:34,414 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:34,414 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-15T14:15:34,415 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T14:15:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-15T14:15:34,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742051_1227 (size=1061) 2024-11-15T14:15:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-15T14:15:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-15T14:15:34,811 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T14:15:34,823 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931 2024-11-15T14:15:34,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742052_1228 (size=53) 2024-11-15T14:15:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-15T14:15:35,231 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:15:35,231 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e966eb2fe52084edc96ca33b9ecedaa2, disabling compactions & flushes 2024-11-15T14:15:35,231 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:35,231 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:35,231 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. after waiting 0 ms 2024-11-15T14:15:35,231 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:35,232 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:35,232 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e966eb2fe52084edc96ca33b9ecedaa2: Waiting for close lock at 1731680135231Disabling compacts and flushes for region at 1731680135231Disabling writes for close at 1731680135231Writing region close event to WAL at 1731680135231Closed at 1731680135231 2024-11-15T14:15:35,234 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T14:15:35,235 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731680135234"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731680135234"}]},"ts":"1731680135234"} 2024-11-15T14:15:35,237 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T14:15:35,239 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T14:15:35,239 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680135239"}]},"ts":"1731680135239"} 2024-11-15T14:15:35,241 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T14:15:35,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e966eb2fe52084edc96ca33b9ecedaa2, ASSIGN}] 2024-11-15T14:15:35,243 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e966eb2fe52084edc96ca33b9ecedaa2, ASSIGN 2024-11-15T14:15:35,244 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e966eb2fe52084edc96ca33b9ecedaa2, ASSIGN; state=OFFLINE, location=26754e923c4a,40177,1731680077530; forceNewPlan=false, retain=false 2024-11-15T14:15:35,395 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=e966eb2fe52084edc96ca33b9ecedaa2, regionState=OPENING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:15:35,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e966eb2fe52084edc96ca33b9ecedaa2, ASSIGN because future has completed 2024-11-15T14:15:35,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:15:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-15T14:15:35,564 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:35,564 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} 2024-11-15T14:15:35,565 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:35,565 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:15:35,566 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:35,566 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:35,568 INFO [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:35,569 INFO [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:15:35,570 INFO [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e966eb2fe52084edc96ca33b9ecedaa2 columnFamilyName A 2024-11-15T14:15:35,570 DEBUG [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:35,571 INFO [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] regionserver.HStore(327): Store=e966eb2fe52084edc96ca33b9ecedaa2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:15:35,571 INFO [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:35,572 INFO [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:15:35,573 INFO [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e966eb2fe52084edc96ca33b9ecedaa2 columnFamilyName B 2024-11-15T14:15:35,573 DEBUG [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:35,574 INFO [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] regionserver.HStore(327): Store=e966eb2fe52084edc96ca33b9ecedaa2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:15:35,574 INFO [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:35,576 INFO [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:15:35,576 INFO [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e966eb2fe52084edc96ca33b9ecedaa2 columnFamilyName C 2024-11-15T14:15:35,576 DEBUG [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:15:35,577 INFO [StoreOpener-e966eb2fe52084edc96ca33b9ecedaa2-1 {}] regionserver.HStore(327): Store=e966eb2fe52084edc96ca33b9ecedaa2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:15:35,577 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:35,577 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:35,578 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:35,578 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:35,579 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:35,579 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:35,580 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T14:15:35,581 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:35,583 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T14:15:35,583 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened e966eb2fe52084edc96ca33b9ecedaa2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71609202, jitterRate=0.06706026196479797}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T14:15:35,583 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:35,584 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for e966eb2fe52084edc96ca33b9ecedaa2: Running coprocessor pre-open hook at 1731680135566Writing region info on filesystem at 1731680135566Initializing all the Stores at 1731680135567 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680135567Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680135567Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680135567Cleaning up temporary data from old regions at 1731680135579 (+12 ms)Running coprocessor post-open hooks at 1731680135583 (+4 ms)Region opened successfully at 1731680135584 (+1 ms) 2024-11-15T14:15:35,584 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., pid=65, masterSystemTime=1731680135556 2024-11-15T14:15:35,586 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:35,586 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:35,587 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=e966eb2fe52084edc96ca33b9ecedaa2, regionState=OPEN, openSeqNum=2, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:15:35,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:15:35,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-15T14:15:35,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 in 187 msec 2024-11-15T14:15:35,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-15T14:15:35,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e966eb2fe52084edc96ca33b9ecedaa2, ASSIGN in 349 msec 2024-11-15T14:15:35,592 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T14:15:35,592 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680135592"}]},"ts":"1731680135592"} 2024-11-15T14:15:35,593 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T14:15:35,594 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T14:15:35,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-11-15T14:15:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-15T14:15:36,563 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:36,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:36,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:36,569 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c3d035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:36,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:36,572 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dfeeed0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,574 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:36,576 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:36,577 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T14:15:36,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 26754e923c4a,35807,1731680075796 2024-11-15T14:15:36,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@944096e 2024-11-15T14:15:36,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T14:15:36,578 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T14:15:36,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:36,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:36,580 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:36,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:36,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ea41ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:36,581 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d451f9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,583 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:36,583 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:36,584 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79853070, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:36,585 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767a9b95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:36,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:36,587 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f6923f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:36,588 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e153ff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,589 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:36,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:36,590 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@211a940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:36,591 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:36,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3644fc97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:36,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:36,593 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2408f04a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:36,595 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cb3d54f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:36,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:36,597 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4449bad9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:36,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:36,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:36,598 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25ceb06f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,600 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:36,600 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:36,600 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@591d8458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:36,602 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:36,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15ea3097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:36,603 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:36,604 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53bdf5a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:36,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:36,605 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4ca728, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,607 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:36,607 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:36,608 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f13d35a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:36,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:36,610 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:36,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699c2d4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,614 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:15:36,614 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:15:36,615 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:15:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:15:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:15:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511df334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:15:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:15:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:36,617 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:15:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d51f064, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:15:36,619 DEBUG [Thread-1072 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:36,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:36,623 DEBUG [Thread-1079 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:36,623 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:36,623 DEBUG [Thread-1081 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:15:36,624 DEBUG [Thread-1085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-15T14:15:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:36,625 DEBUG [Thread-1068 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T14:15:36,625 DEBUG [Thread-1087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:36,626 DEBUG [Thread-1070 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:36,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:36,626 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:36,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:36,626 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:36,627 DEBUG [Thread-1076 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:36,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:36,628 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:36,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:36,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:36,629 DEBUG [Thread-1074 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:36,629 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:15:36,630 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:36,631 DEBUG [Thread-1074 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:36,631 DEBUG [Thread-1083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:15:36,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:15:36,631 DEBUG [Thread-1074 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:36,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:15:36,633 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:15:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:15:36,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:15:36,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:15:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:15:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:15:36,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:15:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:15:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:15:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:36,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T14:15:36,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:36,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:36,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:36,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:36,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:36,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:36,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680196667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:36,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/35c244cc7d7b4c84a4719ed12ca98306 is 50, key is test_row_0/A:col10/1731680136642/Put/seqid=0 2024-11-15T14:15:36,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680136655, max timestamp=1731680136672Num uniques 859; 2024-11-15T14:15:36,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680136655, max timestamp=1731680136672Num uniques 859; 2024-11-15T14:15:36,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:36,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:36,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680136655, max timestamp=1731680136672Num uniques 859; 2024-11-15T14:15:36,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:36,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742053_1229 (size=12001) 2024-11-15T14:15:36,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/35c244cc7d7b4c84a4719ed12ca98306 2024-11-15T14:15:36,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680196703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:36,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680196703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:36,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/7b6bad13e6b34ea7b00dfb9ae1422d7b is 50, key is test_row_0/B:col10/1731680136642/Put/seqid=0 2024-11-15T14:15:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:36,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742054_1230 (size=12001) 2024-11-15T14:15:36,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680196711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:36,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680196712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:36,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/7b6bad13e6b34ea7b00dfb9ae1422d7b 2024-11-15T14:15:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T14:15:36,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/308a8ab5ab494bf88dc4bb442d0c2f42 is 50, key is test_row_0/C:col10/1731680136642/Put/seqid=0 2024-11-15T14:15:36,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742055_1231 (size=12001) 2024-11-15T14:15:36,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680196781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:36,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T14:15:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:36,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:36,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:36,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:36,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:36,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680196822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680196822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680196822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680196822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T14:15:36,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T14:15:36,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:36,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:36,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:36,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:36,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:36,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:36,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680196992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680197032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680197032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680197032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680197033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T14:15:37,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:37,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:37,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:37,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:37,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:37,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/308a8ab5ab494bf88dc4bb442d0c2f42 2024-11-15T14:15:37,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/35c244cc7d7b4c84a4719ed12ca98306 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/35c244cc7d7b4c84a4719ed12ca98306 2024-11-15T14:15:37,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/35c244cc7d7b4c84a4719ed12ca98306, entries=150, sequenceid=12, filesize=11.7 K 2024-11-15T14:15:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/7b6bad13e6b34ea7b00dfb9ae1422d7b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/7b6bad13e6b34ea7b00dfb9ae1422d7b 2024-11-15T14:15:37,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/7b6bad13e6b34ea7b00dfb9ae1422d7b, entries=150, sequenceid=12, filesize=11.7 K 2024-11-15T14:15:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/308a8ab5ab494bf88dc4bb442d0c2f42 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/308a8ab5ab494bf88dc4bb442d0c2f42 2024-11-15T14:15:37,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/308a8ab5ab494bf88dc4bb442d0c2f42, entries=150, sequenceid=12, filesize=11.7 K 2024-11-15T14:15:37,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for e966eb2fe52084edc96ca33b9ecedaa2 in 545ms, sequenceid=12, compaction requested=false 2024-11-15T14:15:37,187 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T14:15:37,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T14:15:37,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T14:15:37,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:37,265 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-15T14:15:37,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:37,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:37,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:37,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:37,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:37,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/03c68fc8a6154b6cb3511553b4dd265a is 50, key is test_row_0/A:col10/1731680136709/Put/seqid=0 2024-11-15T14:15:37,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742056_1232 (size=12001) 2024-11-15T14:15:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:37,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:37,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680197337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680197342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680197342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680197342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680197342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680197452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680197662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,675 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/03c68fc8a6154b6cb3511553b4dd265a 2024-11-15T14:15:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/064b6abadc06454d8a839654738c7398 is 50, key is test_row_0/B:col10/1731680136709/Put/seqid=0 2024-11-15T14:15:37,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742057_1233 (size=12001) 2024-11-15T14:15:37,687 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/064b6abadc06454d8a839654738c7398 2024-11-15T14:15:37,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/f216676024b7476eacf2c254f9aa0ae2 is 50, key is test_row_0/C:col10/1731680136709/Put/seqid=0 2024-11-15T14:15:37,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742058_1234 (size=12001) 2024-11-15T14:15:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T14:15:37,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680197851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680197851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680197851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680197852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:37,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680197972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:38,102 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/f216676024b7476eacf2c254f9aa0ae2 2024-11-15T14:15:38,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/03c68fc8a6154b6cb3511553b4dd265a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/03c68fc8a6154b6cb3511553b4dd265a 2024-11-15T14:15:38,133 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/03c68fc8a6154b6cb3511553b4dd265a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T14:15:38,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/064b6abadc06454d8a839654738c7398 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/064b6abadc06454d8a839654738c7398 2024-11-15T14:15:38,150 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/064b6abadc06454d8a839654738c7398, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T14:15:38,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/f216676024b7476eacf2c254f9aa0ae2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/f216676024b7476eacf2c254f9aa0ae2 2024-11-15T14:15:38,161 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/f216676024b7476eacf2c254f9aa0ae2, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T14:15:38,162 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for e966eb2fe52084edc96ca33b9ecedaa2 in 897ms, sequenceid=49, compaction requested=false 2024-11-15T14:15:38,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:38,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:38,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-15T14:15:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-15T14:15:38,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-15T14:15:38,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5360 sec 2024-11-15T14:15:38,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.5430 sec 2024-11-15T14:15:38,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T14:15:38,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:38,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:38,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:38,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:38,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:38,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:38,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/02af1b821e344d078f76365bc1a73347 is 50, key is test_row_0/A:col10/1731680138486/Put/seqid=0 2024-11-15T14:15:38,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742059_1235 (size=9657) 2024-11-15T14:15:38,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/02af1b821e344d078f76365bc1a73347 2024-11-15T14:15:38,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/7fdcb3c4001744299aabb228496da9b3 is 50, key is test_row_0/B:col10/1731680138486/Put/seqid=0 2024-11-15T14:15:38,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742060_1236 (size=9657) 2024-11-15T14:15:38,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/7fdcb3c4001744299aabb228496da9b3 2024-11-15T14:15:38,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680138580, max timestamp=1731680138628Num uniques 859; 2024-11-15T14:15:38,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:38,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680138580, max timestamp=1731680138628Num uniques 859; 2024-11-15T14:15:38,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:38,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/f4f4e4a4a5344b799309aef8e31d9b5b is 50, key is test_row_0/C:col10/1731680138486/Put/seqid=0 2024-11-15T14:15:38,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680138580, max timestamp=1731680138628Num uniques 859; 2024-11-15T14:15:38,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:38,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742061_1237 (size=9657) 2024-11-15T14:15:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T14:15:38,783 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:38,784 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-15T14:15:38,791 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T14:15:38,792 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:38,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:38,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680198791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:38,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680198861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:38,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680198861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:38,865 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in e966eb2fe52084edc96ca33b9ecedaa2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:38,866 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in e966eb2fe52084edc96ca33b9ecedaa2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:38,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680198872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:38,874 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in e966eb2fe52084edc96ca33b9ecedaa2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680198872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:38,879 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in e966eb2fe52084edc96ca33b9ecedaa2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T14:15:38,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680198902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:38,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:38,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:38,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:38,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T14:15:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:38,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:38,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:39,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/f4f4e4a4a5344b799309aef8e31d9b5b 2024-11-15T14:15:39,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T14:15:39,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:39,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:39,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:39,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:39,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T14:15:39,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/02af1b821e344d078f76365bc1a73347 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/02af1b821e344d078f76365bc1a73347 2024-11-15T14:15:39,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/02af1b821e344d078f76365bc1a73347, entries=100, sequenceid=60, filesize=9.4 K 2024-11-15T14:15:39,121 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-15T14:15:39,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:39,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/7fdcb3c4001744299aabb228496da9b3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/7fdcb3c4001744299aabb228496da9b3 2024-11-15T14:15:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680199113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:39,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/7fdcb3c4001744299aabb228496da9b3, entries=100, sequenceid=60, filesize=9.4 K 2024-11-15T14:15:39,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/f4f4e4a4a5344b799309aef8e31d9b5b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/f4f4e4a4a5344b799309aef8e31d9b5b 2024-11-15T14:15:39,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/f4f4e4a4a5344b799309aef8e31d9b5b, entries=100, sequenceid=60, filesize=9.4 K 2024-11-15T14:15:39,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e966eb2fe52084edc96ca33b9ecedaa2 in 673ms, sequenceid=60, compaction requested=true 2024-11-15T14:15:39,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:39,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:39,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:39,168 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:39,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:39,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:39,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:39,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:15:39,169 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:39,170 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:39,170 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/A is initiating minor compaction (all files) 2024-11-15T14:15:39,170 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/A in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:39,171 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/35c244cc7d7b4c84a4719ed12ca98306, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/03c68fc8a6154b6cb3511553b4dd265a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/02af1b821e344d078f76365bc1a73347] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=32.9 K 2024-11-15T14:15:39,171 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:39,171 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/B is initiating minor compaction (all files) 2024-11-15T14:15:39,171 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/B in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:39,171 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 35c244cc7d7b4c84a4719ed12ca98306, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731680136638 2024-11-15T14:15:39,171 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/7b6bad13e6b34ea7b00dfb9ae1422d7b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/064b6abadc06454d8a839654738c7398, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/7fdcb3c4001744299aabb228496da9b3] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=32.9 K 2024-11-15T14:15:39,171 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c68fc8a6154b6cb3511553b4dd265a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731680136700 2024-11-15T14:15:39,171 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b6bad13e6b34ea7b00dfb9ae1422d7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731680136638 2024-11-15T14:15:39,172 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 02af1b821e344d078f76365bc1a73347, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731680137324 2024-11-15T14:15:39,172 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 064b6abadc06454d8a839654738c7398, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731680136700 2024-11-15T14:15:39,172 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fdcb3c4001744299aabb228496da9b3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731680137324 2024-11-15T14:15:39,197 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#A#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:39,198 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/f10a57d90fff496199af5d7b8802a552 is 50, key is test_row_0/A:col10/1731680138486/Put/seqid=0 2024-11-15T14:15:39,209 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:39,209 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/dee74df3465d4329b7d2b6e4a42f6959 is 50, key is test_row_0/B:col10/1731680138486/Put/seqid=0 2024-11-15T14:15:39,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T14:15:39,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:39,255 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-15T14:15:39,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:39,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:39,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:39,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742062_1238 (size=12104) 2024-11-15T14:15:39,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742063_1239 (size=12104) 2024-11-15T14:15:39,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/525f2bd0698940ca85318bf6f9e6415a is 50, key is test_row_0/A:col10/1731680138786/Put/seqid=0 2024-11-15T14:15:39,297 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/dee74df3465d4329b7d2b6e4a42f6959 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/dee74df3465d4329b7d2b6e4a42f6959 2024-11-15T14:15:39,308 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/B of e966eb2fe52084edc96ca33b9ecedaa2 into dee74df3465d4329b7d2b6e4a42f6959(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:39,308 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:39,308 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/B, priority=13, startTime=1731680139168; duration=0sec 2024-11-15T14:15:39,308 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:39,308 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:B 2024-11-15T14:15:39,308 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:39,310 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:39,310 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/C is initiating minor compaction (all files) 2024-11-15T14:15:39,310 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/C in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:39,310 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/308a8ab5ab494bf88dc4bb442d0c2f42, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/f216676024b7476eacf2c254f9aa0ae2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/f4f4e4a4a5344b799309aef8e31d9b5b] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=32.9 K 2024-11-15T14:15:39,311 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 308a8ab5ab494bf88dc4bb442d0c2f42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731680136638 2024-11-15T14:15:39,311 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f216676024b7476eacf2c254f9aa0ae2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731680136700 2024-11-15T14:15:39,312 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4f4e4a4a5344b799309aef8e31d9b5b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731680137324 2024-11-15T14:15:39,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742064_1240 (size=12001) 2024-11-15T14:15:39,326 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/525f2bd0698940ca85318bf6f9e6415a 2024-11-15T14:15:39,329 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#C#compaction#197 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:39,330 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/50ffdcf29d0549069702e1b2d4872bd2 is 50, key is test_row_0/C:col10/1731680138486/Put/seqid=0 2024-11-15T14:15:39,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/270c3d4506c049c8991686cd3842c4e9 is 50, key is test_row_0/B:col10/1731680138786/Put/seqid=0 2024-11-15T14:15:39,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742065_1241 (size=12104) 2024-11-15T14:15:39,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742066_1242 (size=12001) 2024-11-15T14:15:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T14:15:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:39,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680199517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:39,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680199632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:39,681 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/f10a57d90fff496199af5d7b8802a552 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f10a57d90fff496199af5d7b8802a552 2024-11-15T14:15:39,683 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T14:15:39,695 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/A of e966eb2fe52084edc96ca33b9ecedaa2 into f10a57d90fff496199af5d7b8802a552(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:39,695 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:39,695 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/A, priority=13, startTime=1731680139168; duration=0sec 2024-11-15T14:15:39,695 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:39,695 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:A 2024-11-15T14:15:39,791 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/50ffdcf29d0549069702e1b2d4872bd2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/50ffdcf29d0549069702e1b2d4872bd2 2024-11-15T14:15:39,799 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/C of e966eb2fe52084edc96ca33b9ecedaa2 into 50ffdcf29d0549069702e1b2d4872bd2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:39,800 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:39,800 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/C, priority=13, startTime=1731680139168; duration=0sec 2024-11-15T14:15:39,800 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:39,800 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:C 2024-11-15T14:15:39,815 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/270c3d4506c049c8991686cd3842c4e9 2024-11-15T14:15:39,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ad6d61c70719499585bd0d9a164349a1 is 50, key is test_row_0/C:col10/1731680138786/Put/seqid=0 2024-11-15T14:15:39,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680199854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:39,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:39,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:39,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:39,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742067_1243 (size=12001) 2024-11-15T14:15:39,871 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ad6d61c70719499585bd0d9a164349a1 2024-11-15T14:15:39,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/525f2bd0698940ca85318bf6f9e6415a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/525f2bd0698940ca85318bf6f9e6415a 2024-11-15T14:15:39,885 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/525f2bd0698940ca85318bf6f9e6415a, entries=150, sequenceid=96, filesize=11.7 K 2024-11-15T14:15:39,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/270c3d4506c049c8991686cd3842c4e9 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/270c3d4506c049c8991686cd3842c4e9 2024-11-15T14:15:39,905 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/270c3d4506c049c8991686cd3842c4e9, entries=150, sequenceid=96, filesize=11.7 K 2024-11-15T14:15:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ad6d61c70719499585bd0d9a164349a1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ad6d61c70719499585bd0d9a164349a1 2024-11-15T14:15:39,915 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ad6d61c70719499585bd0d9a164349a1, entries=150, sequenceid=96, filesize=11.7 K 2024-11-15T14:15:39,918 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for e966eb2fe52084edc96ca33b9ecedaa2 in 664ms, sequenceid=96, compaction requested=false 2024-11-15T14:15:39,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:39,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:39,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-15T14:15:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-15T14:15:39,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-15T14:15:39,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1270 sec 2024-11-15T14:15:39,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1420 sec 2024-11-15T14:15:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T14:15:39,933 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:39,934 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-15T14:15:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T14:15:39,939 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:39,944 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:39,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T14:15:40,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T14:15:40,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:40,097 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T14:15:40,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:40,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:40,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:40,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:40,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:40,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:40,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/f60020252bf14dd382526e126530522d is 50, key is test_row_0/A:col10/1731680139502/Put/seqid=0 2024-11-15T14:15:40,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742068_1244 (size=12001) 2024-11-15T14:15:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:40,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T14:15:40,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680140295, max timestamp=1731680140347Num uniques 859; 2024-11-15T14:15:40,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:40,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680140295, max timestamp=1731680140347Num uniques 859; 2024-11-15T14:15:40,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:40,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680140295, max timestamp=1731680140347Num uniques 859; 2024-11-15T14:15:40,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:40,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680200489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:40,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T14:15:40,562 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/f60020252bf14dd382526e126530522d 2024-11-15T14:15:40,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/84629ba08b8146e99f99b30a70db156f is 50, key is test_row_0/B:col10/1731680139502/Put/seqid=0 2024-11-15T14:15:40,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680200602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:40,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742069_1245 (size=12001) 2024-11-15T14:15:40,625 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/84629ba08b8146e99f99b30a70db156f 2024-11-15T14:15:40,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/6df4b46d799b42e0888cd615d8dc02a1 is 50, key is test_row_0/C:col10/1731680139502/Put/seqid=0 2024-11-15T14:15:40,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742070_1246 (size=12001) 2024-11-15T14:15:40,712 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/6df4b46d799b42e0888cd615d8dc02a1 2024-11-15T14:15:40,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/f60020252bf14dd382526e126530522d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f60020252bf14dd382526e126530522d 2024-11-15T14:15:40,737 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f60020252bf14dd382526e126530522d, entries=150, sequenceid=110, filesize=11.7 K 2024-11-15T14:15:40,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/84629ba08b8146e99f99b30a70db156f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/84629ba08b8146e99f99b30a70db156f 2024-11-15T14:15:40,743 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/84629ba08b8146e99f99b30a70db156f, entries=150, sequenceid=110, filesize=11.7 K 2024-11-15T14:15:40,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/6df4b46d799b42e0888cd615d8dc02a1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/6df4b46d799b42e0888cd615d8dc02a1 2024-11-15T14:15:40,751 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/6df4b46d799b42e0888cd615d8dc02a1, entries=150, sequenceid=110, filesize=11.7 K 2024-11-15T14:15:40,754 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e966eb2fe52084edc96ca33b9ecedaa2 in 657ms, sequenceid=110, compaction requested=true 2024-11-15T14:15:40,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:40,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:40,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-15T14:15:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-15T14:15:40,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-15T14:15:40,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 813 msec 2024-11-15T14:15:40,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 826 msec 2024-11-15T14:15:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:40,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-15T14:15:40,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:40,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:40,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:40,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:40,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:40,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:40,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/8cf038b420224a278aa030ed68713adf is 50, key is test_row_0/A:col10/1731680140460/Put/seqid=0 2024-11-15T14:15:40,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742071_1247 (size=14541) 2024-11-15T14:15:40,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/8cf038b420224a278aa030ed68713adf 2024-11-15T14:15:40,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:40,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680200882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680200882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:40,890 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in e966eb2fe52084edc96ca33b9ecedaa2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:40,892 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in e966eb2fe52084edc96ca33b9ecedaa2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:40,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680200889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:40,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:40,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680200892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:40,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:40,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680200892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:40,899 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in e966eb2fe52084edc96ca33b9ecedaa2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:40,900 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in e966eb2fe52084edc96ca33b9ecedaa2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:40,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/83b6bfad6570489e9cd3168add9b33c2 is 50, key is test_row_0/B:col10/1731680140460/Put/seqid=0 2024-11-15T14:15:40,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742072_1248 (size=12151) 2024-11-15T14:15:41,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680201003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:41,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:41,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:41,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T14:15:41,073 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:41,074 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-15T14:15:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T14:15:41,079 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:41,080 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:41,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T14:15:41,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680201212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:41,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T14:15:41,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:41,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:41,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:41,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/83b6bfad6570489e9cd3168add9b33c2 2024-11-15T14:15:41,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/2d194672ec0847c68e13b70184e51293 is 50, key is test_row_0/C:col10/1731680140460/Put/seqid=0 2024-11-15T14:15:41,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T14:15:41,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:41,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:41,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:41,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T14:15:41,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742073_1249 (size=12151) 2024-11-15T14:15:41,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/2d194672ec0847c68e13b70184e51293 2024-11-15T14:15:41,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/8cf038b420224a278aa030ed68713adf as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/8cf038b420224a278aa030ed68713adf 2024-11-15T14:15:41,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/8cf038b420224a278aa030ed68713adf, entries=200, sequenceid=147, filesize=14.2 K 2024-11-15T14:15:41,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/83b6bfad6570489e9cd3168add9b33c2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/83b6bfad6570489e9cd3168add9b33c2 2024-11-15T14:15:41,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/83b6bfad6570489e9cd3168add9b33c2, entries=150, sequenceid=147, filesize=11.9 K 2024-11-15T14:15:41,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/2d194672ec0847c68e13b70184e51293 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/2d194672ec0847c68e13b70184e51293 2024-11-15T14:15:41,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/2d194672ec0847c68e13b70184e51293, entries=150, sequenceid=147, filesize=11.9 K 2024-11-15T14:15:41,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for e966eb2fe52084edc96ca33b9ecedaa2 in 616ms, sequenceid=147, compaction requested=true 2024-11-15T14:15:41,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:41,440 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:41,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:41,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:41,441 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:41,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:41,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:41,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:41,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:41,442 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50647 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:41,442 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/A is initiating minor compaction (all files) 2024-11-15T14:15:41,442 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/A in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:41,442 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f10a57d90fff496199af5d7b8802a552, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/525f2bd0698940ca85318bf6f9e6415a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f60020252bf14dd382526e126530522d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/8cf038b420224a278aa030ed68713adf] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=49.5 K 2024-11-15T14:15:41,443 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:41,443 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f10a57d90fff496199af5d7b8802a552, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731680136700 2024-11-15T14:15:41,443 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/B is initiating minor compaction (all files) 2024-11-15T14:15:41,443 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/B in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:41,443 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/dee74df3465d4329b7d2b6e4a42f6959, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/270c3d4506c049c8991686cd3842c4e9, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/84629ba08b8146e99f99b30a70db156f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/83b6bfad6570489e9cd3168add9b33c2] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=47.1 K 2024-11-15T14:15:41,444 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 525f2bd0698940ca85318bf6f9e6415a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731680138739 2024-11-15T14:15:41,444 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting dee74df3465d4329b7d2b6e4a42f6959, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731680136700 2024-11-15T14:15:41,444 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f60020252bf14dd382526e126530522d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731680139462 2024-11-15T14:15:41,444 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 270c3d4506c049c8991686cd3842c4e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731680138739 2024-11-15T14:15:41,444 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cf038b420224a278aa030ed68713adf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731680140460 2024-11-15T14:15:41,444 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 84629ba08b8146e99f99b30a70db156f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731680139462 2024-11-15T14:15:41,445 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 83b6bfad6570489e9cd3168add9b33c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731680140460 2024-11-15T14:15:41,467 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#A#compaction#206 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:41,468 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/0c3868fc35e94cc68b9f1991b1e0bf2d is 50, key is test_row_0/A:col10/1731680140460/Put/seqid=0 2024-11-15T14:15:41,473 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#B#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:41,476 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/87613ffb399e4c659bd162f9661d8b34 is 50, key is test_row_0/B:col10/1731680140460/Put/seqid=0 2024-11-15T14:15:41,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742074_1250 (size=12391) 2024-11-15T14:15:41,526 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/0c3868fc35e94cc68b9f1991b1e0bf2d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0c3868fc35e94cc68b9f1991b1e0bf2d 2024-11-15T14:15:41,537 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/A of e966eb2fe52084edc96ca33b9ecedaa2 into 0c3868fc35e94cc68b9f1991b1e0bf2d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:41,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T14:15:41,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:41,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:41,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:41,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:41,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:41,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:41,539 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:41,539 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/A, priority=12, startTime=1731680141440; duration=0sec 2024-11-15T14:15:41,540 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:41,540 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:A 2024-11-15T14:15:41,540 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:41,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742075_1251 (size=12391) 2024-11-15T14:15:41,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T14:15:41,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:41,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:41,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:41,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/d15d9cdfeee14cbe9f3860bb3d6c2cb1 is 50, key is test_row_0/A:col10/1731680140832/Put/seqid=0 2024-11-15T14:15:41,555 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:41,556 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/C is initiating minor compaction (all files) 2024-11-15T14:15:41,556 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/C in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:41,556 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/50ffdcf29d0549069702e1b2d4872bd2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ad6d61c70719499585bd0d9a164349a1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/6df4b46d799b42e0888cd615d8dc02a1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/2d194672ec0847c68e13b70184e51293] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=47.1 K 2024-11-15T14:15:41,558 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50ffdcf29d0549069702e1b2d4872bd2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731680136700 2024-11-15T14:15:41,558 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/87613ffb399e4c659bd162f9661d8b34 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/87613ffb399e4c659bd162f9661d8b34 2024-11-15T14:15:41,558 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad6d61c70719499585bd0d9a164349a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731680138739 2024-11-15T14:15:41,565 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/B of e966eb2fe52084edc96ca33b9ecedaa2 into 87613ffb399e4c659bd162f9661d8b34(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:41,565 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:41,565 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/B, priority=12, startTime=1731680141441; duration=0sec 2024-11-15T14:15:41,565 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:41,565 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:B 2024-11-15T14:15:41,566 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df4b46d799b42e0888cd615d8dc02a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731680139462 2024-11-15T14:15:41,570 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d194672ec0847c68e13b70184e51293, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731680140460 2024-11-15T14:15:41,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742076_1252 (size=12151) 2024-11-15T14:15:41,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/d15d9cdfeee14cbe9f3860bb3d6c2cb1 2024-11-15T14:15:41,637 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:41,637 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/454c2ad055b44b2cb0254423d1524ce5 is 50, key is test_row_0/C:col10/1731680140460/Put/seqid=0 2024-11-15T14:15:41,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/e4eddf55544c46c8b1d6e5271def4f13 is 50, key is test_row_0/B:col10/1731680140832/Put/seqid=0 2024-11-15T14:15:41,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680141648, max timestamp=1731680141679Num uniques 859; 2024-11-15T14:15:41,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680141648, max timestamp=1731680141679Num uniques 859; 2024-11-15T14:15:41,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:41,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:41,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680141648, max timestamp=1731680141679Num uniques 859; 2024-11-15T14:15:41,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:15:41,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742077_1253 (size=12391) 2024-11-15T14:15:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T14:15:41,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T14:15:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:41,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,717 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/454c2ad055b44b2cb0254423d1524ce5 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/454c2ad055b44b2cb0254423d1524ce5 2024-11-15T14:15:41,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742078_1254 (size=12151) 2024-11-15T14:15:41,732 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/C of e966eb2fe52084edc96ca33b9ecedaa2 into 454c2ad055b44b2cb0254423d1524ce5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:41,732 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:41,732 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/C, priority=12, startTime=1731680141441; duration=0sec 2024-11-15T14:15:41,732 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:41,732 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:C 2024-11-15T14:15:41,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T14:15:41,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:41,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:41,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680201879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:41,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680201992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:42,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T14:15:42,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:42,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:42,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:42,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:42,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:42,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/e4eddf55544c46c8b1d6e5271def4f13 2024-11-15T14:15:42,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/01e55b1df0154ba48d1c08101b243b5e is 50, key is test_row_0/C:col10/1731680140832/Put/seqid=0 2024-11-15T14:15:42,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T14:15:42,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:42,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:42,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:42,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:42,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:42,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742079_1255 (size=12151) 2024-11-15T14:15:42,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/01e55b1df0154ba48d1c08101b243b5e 2024-11-15T14:15:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/d15d9cdfeee14cbe9f3860bb3d6c2cb1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/d15d9cdfeee14cbe9f3860bb3d6c2cb1 2024-11-15T14:15:42,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/d15d9cdfeee14cbe9f3860bb3d6c2cb1, entries=150, sequenceid=159, filesize=11.9 K 2024-11-15T14:15:42,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/e4eddf55544c46c8b1d6e5271def4f13 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e4eddf55544c46c8b1d6e5271def4f13 2024-11-15T14:15:42,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e4eddf55544c46c8b1d6e5271def4f13, entries=150, sequenceid=159, filesize=11.9 K 2024-11-15T14:15:42,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:42,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/01e55b1df0154ba48d1c08101b243b5e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/01e55b1df0154ba48d1c08101b243b5e 2024-11-15T14:15:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680202202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:42,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:42,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:42,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:42,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/01e55b1df0154ba48d1c08101b243b5e, entries=150, sequenceid=159, filesize=11.9 K 2024-11-15T14:15:42,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e966eb2fe52084edc96ca33b9ecedaa2 in 674ms, sequenceid=159, compaction requested=false 2024-11-15T14:15:42,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T14:15:42,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T14:15:42,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:42,336 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-15T14:15:42,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:42,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:42,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:42,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:42,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:42,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:42,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/87bd88b592ee43f6b6745fe9f042be17 is 50, key is test_row_0/A:col10/1731680141860/Put/seqid=0 2024-11-15T14:15:42,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742080_1256 (size=12151) 2024-11-15T14:15:42,409 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/87bd88b592ee43f6b6745fe9f042be17 2024-11-15T14:15:42,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/82482276eaba4199984eda2a77ab8b9f is 50, key is test_row_0/B:col10/1731680141860/Put/seqid=0 2024-11-15T14:15:42,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742081_1257 (size=12151) 2024-11-15T14:15:42,499 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/82482276eaba4199984eda2a77ab8b9f 2024-11-15T14:15:42,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:42,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/0f5d1fa3551a470e911037b1b5513466 is 50, key is test_row_0/C:col10/1731680141860/Put/seqid=0 2024-11-15T14:15:42,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742082_1258 (size=12151) 2024-11-15T14:15:42,565 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/0f5d1fa3551a470e911037b1b5513466 2024-11-15T14:15:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/87bd88b592ee43f6b6745fe9f042be17 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/87bd88b592ee43f6b6745fe9f042be17 2024-11-15T14:15:42,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680202574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:42,597 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/87bd88b592ee43f6b6745fe9f042be17, entries=150, sequenceid=197, filesize=11.9 K 2024-11-15T14:15:42,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/82482276eaba4199984eda2a77ab8b9f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/82482276eaba4199984eda2a77ab8b9f 2024-11-15T14:15:42,611 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/82482276eaba4199984eda2a77ab8b9f, entries=150, sequenceid=197, filesize=11.9 K 2024-11-15T14:15:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/0f5d1fa3551a470e911037b1b5513466 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/0f5d1fa3551a470e911037b1b5513466 2024-11-15T14:15:42,624 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/0f5d1fa3551a470e911037b1b5513466, entries=150, sequenceid=197, filesize=11.9 K 2024-11-15T14:15:42,626 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for e966eb2fe52084edc96ca33b9ecedaa2 in 289ms, sequenceid=197, compaction requested=true 2024-11-15T14:15:42,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:42,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:42,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-15T14:15:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-15T14:15:42,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-15T14:15:42,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5560 sec 2024-11-15T14:15:42,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.5670 sec 2024-11-15T14:15:42,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T14:15:42,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:42,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:42,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:42,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:42,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:42,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:42,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/fd940a19a75246d08d0e063c20ea8ca8 is 50, key is test_row_0/A:col10/1731680142556/Put/seqid=0 2024-11-15T14:15:42,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742083_1259 (size=14541) 2024-11-15T14:15:42,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/fd940a19a75246d08d0e063c20ea8ca8 2024-11-15T14:15:42,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/469109e67dbe4d9faa43451d2d0ef5f6 is 50, key is test_row_0/B:col10/1731680142556/Put/seqid=0 2024-11-15T14:15:42,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742084_1260 (size=12151) 2024-11-15T14:15:42,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/469109e67dbe4d9faa43451d2d0ef5f6 2024-11-15T14:15:42,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:42,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680202866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:42,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/1ec7e00015ef493f9a43ba58e8312c9f is 50, key is test_row_0/C:col10/1731680142556/Put/seqid=0 2024-11-15T14:15:42,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742085_1261 (size=12151) 2024-11-15T14:15:42,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680202982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:43,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680203194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:43,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T14:15:43,222 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:43,222 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-15T14:15:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T14:15:43,229 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:43,231 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:43,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T14:15:43,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/1ec7e00015ef493f9a43ba58e8312c9f 2024-11-15T14:15:43,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/fd940a19a75246d08d0e063c20ea8ca8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/fd940a19a75246d08d0e063c20ea8ca8 2024-11-15T14:15:43,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/fd940a19a75246d08d0e063c20ea8ca8, entries=200, sequenceid=209, filesize=14.2 K 2024-11-15T14:15:43,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/469109e67dbe4d9faa43451d2d0ef5f6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/469109e67dbe4d9faa43451d2d0ef5f6 2024-11-15T14:15:43,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/469109e67dbe4d9faa43451d2d0ef5f6, entries=150, sequenceid=209, filesize=11.9 K 2024-11-15T14:15:43,367 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-15T14:15:43,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/1ec7e00015ef493f9a43ba58e8312c9f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/1ec7e00015ef493f9a43ba58e8312c9f 2024-11-15T14:15:43,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/1ec7e00015ef493f9a43ba58e8312c9f, entries=150, sequenceid=209, filesize=11.9 K 2024-11-15T14:15:43,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e966eb2fe52084edc96ca33b9ecedaa2 in 682ms, sequenceid=209, compaction requested=true 2024-11-15T14:15:43,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:43,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:43,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:43,378 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:43,378 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:43,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:43,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:43,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:43,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:43,381 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51234 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:43,381 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/A is initiating minor compaction (all files) 2024-11-15T14:15:43,381 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/A in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:43,381 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0c3868fc35e94cc68b9f1991b1e0bf2d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/d15d9cdfeee14cbe9f3860bb3d6c2cb1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/87bd88b592ee43f6b6745fe9f042be17, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/fd940a19a75246d08d0e063c20ea8ca8] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=50.0 K 2024-11-15T14:15:43,382 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:43,382 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/B is initiating minor compaction (all files) 2024-11-15T14:15:43,382 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/B in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:43,382 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/87613ffb399e4c659bd162f9661d8b34, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e4eddf55544c46c8b1d6e5271def4f13, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/82482276eaba4199984eda2a77ab8b9f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/469109e67dbe4d9faa43451d2d0ef5f6] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=47.7 K 2024-11-15T14:15:43,382 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c3868fc35e94cc68b9f1991b1e0bf2d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731680140460 2024-11-15T14:15:43,382 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 87613ffb399e4c659bd162f9661d8b34, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731680140460 2024-11-15T14:15:43,382 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting d15d9cdfeee14cbe9f3860bb3d6c2cb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731680140832 2024-11-15T14:15:43,383 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting e4eddf55544c46c8b1d6e5271def4f13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731680140832 2024-11-15T14:15:43,383 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87bd88b592ee43f6b6745fe9f042be17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731680141827 2024-11-15T14:15:43,383 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 82482276eaba4199984eda2a77ab8b9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731680141827 2024-11-15T14:15:43,383 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd940a19a75246d08d0e063c20ea8ca8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731680142515 2024-11-15T14:15:43,384 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 469109e67dbe4d9faa43451d2d0ef5f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731680142556 2024-11-15T14:15:43,397 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#B#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:43,398 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/12f245dd7efe480eb3797cfac13fa320 is 50, key is test_row_0/B:col10/1731680142556/Put/seqid=0 2024-11-15T14:15:43,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T14:15:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:43,405 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T14:15:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:43,409 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:43,410 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/82233b87b21f458d9abd963bb606c408 is 50, key is test_row_0/A:col10/1731680142556/Put/seqid=0 2024-11-15T14:15:43,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/55814f091ad8412d90dcb06aad0fc6e4 is 50, key is test_row_0/A:col10/1731680142862/Put/seqid=0 2024-11-15T14:15:43,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742086_1262 (size=12527) 2024-11-15T14:15:43,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742087_1263 (size=12527) 2024-11-15T14:15:43,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:43,509 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/82233b87b21f458d9abd963bb606c408 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/82233b87b21f458d9abd963bb606c408 2024-11-15T14:15:43,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742088_1264 (size=12151) 2024-11-15T14:15:43,520 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/55814f091ad8412d90dcb06aad0fc6e4 2024-11-15T14:15:43,521 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/A of e966eb2fe52084edc96ca33b9ecedaa2 into 82233b87b21f458d9abd963bb606c408(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:43,521 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:43,521 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/A, priority=12, startTime=1731680143378; duration=0sec 2024-11-15T14:15:43,522 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:43,522 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:A 2024-11-15T14:15:43,522 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:43,524 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:43,524 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/C is initiating minor compaction (all files) 2024-11-15T14:15:43,524 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/C in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:43,524 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/454c2ad055b44b2cb0254423d1524ce5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/01e55b1df0154ba48d1c08101b243b5e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/0f5d1fa3551a470e911037b1b5513466, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/1ec7e00015ef493f9a43ba58e8312c9f] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=47.7 K 2024-11-15T14:15:43,525 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 454c2ad055b44b2cb0254423d1524ce5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731680140460 2024-11-15T14:15:43,525 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01e55b1df0154ba48d1c08101b243b5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731680140832 2024-11-15T14:15:43,525 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f5d1fa3551a470e911037b1b5513466, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731680141827 2024-11-15T14:15:43,526 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ec7e00015ef493f9a43ba58e8312c9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731680142556 2024-11-15T14:15:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T14:15:43,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/e2dceb32ff3844419881eed39dba2968 is 50, key is test_row_0/B:col10/1731680142862/Put/seqid=0 2024-11-15T14:15:43,586 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#C#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:43,587 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/9e6441ccbdd749f2af0b6cd41fbf8b6c is 50, key is test_row_0/C:col10/1731680142556/Put/seqid=0 2024-11-15T14:15:43,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680203590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:43,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742089_1265 (size=12151) 2024-11-15T14:15:43,619 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/e2dceb32ff3844419881eed39dba2968 2024-11-15T14:15:43,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742090_1266 (size=12527) 2024-11-15T14:15:43,652 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/9e6441ccbdd749f2af0b6cd41fbf8b6c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/9e6441ccbdd749f2af0b6cd41fbf8b6c 2024-11-15T14:15:43,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/2d5a69202a044c0d88e467a9eb5b6f4d is 50, key is test_row_0/C:col10/1731680142862/Put/seqid=0 2024-11-15T14:15:43,661 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/C of e966eb2fe52084edc96ca33b9ecedaa2 into 9e6441ccbdd749f2af0b6cd41fbf8b6c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:43,661 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:43,661 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/C, priority=12, startTime=1731680143379; duration=0sec 2024-11-15T14:15:43,661 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:43,662 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:C 2024-11-15T14:15:43,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742091_1267 (size=12151) 2024-11-15T14:15:43,693 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/2d5a69202a044c0d88e467a9eb5b6f4d 2024-11-15T14:15:43,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/55814f091ad8412d90dcb06aad0fc6e4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/55814f091ad8412d90dcb06aad0fc6e4 2024-11-15T14:15:43,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680203702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:43,720 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/55814f091ad8412d90dcb06aad0fc6e4, entries=150, sequenceid=229, filesize=11.9 K 2024-11-15T14:15:43,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/e2dceb32ff3844419881eed39dba2968 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e2dceb32ff3844419881eed39dba2968 2024-11-15T14:15:43,731 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e2dceb32ff3844419881eed39dba2968, entries=150, sequenceid=229, filesize=11.9 K 2024-11-15T14:15:43,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/2d5a69202a044c0d88e467a9eb5b6f4d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/2d5a69202a044c0d88e467a9eb5b6f4d 2024-11-15T14:15:43,744 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/2d5a69202a044c0d88e467a9eb5b6f4d, entries=150, sequenceid=229, filesize=11.9 K 2024-11-15T14:15:43,745 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e966eb2fe52084edc96ca33b9ecedaa2 in 340ms, sequenceid=229, compaction requested=false 2024-11-15T14:15:43,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:43,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:43,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-15T14:15:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-15T14:15:43,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-15T14:15:43,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 517 msec 2024-11-15T14:15:43,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 529 msec 2024-11-15T14:15:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T14:15:43,852 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:43,853 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-15T14:15:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T14:15:43,856 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:43,857 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:43,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:43,861 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/12f245dd7efe480eb3797cfac13fa320 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/12f245dd7efe480eb3797cfac13fa320 2024-11-15T14:15:43,868 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/B of e966eb2fe52084edc96ca33b9ecedaa2 into 12f245dd7efe480eb3797cfac13fa320(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:43,868 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:43,868 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/B, priority=12, startTime=1731680143378; duration=0sec 2024-11-15T14:15:43,868 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:43,868 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:B 2024-11-15T14:15:43,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T14:15:43,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:43,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:43,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:43,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:43,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:43,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:43,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/ce224d9b10294a6cb295d550f2ec6bc8 is 50, key is test_row_0/A:col10/1731680143518/Put/seqid=0 2024-11-15T14:15:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T14:15:43,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742092_1268 (size=12151) 2024-11-15T14:15:43,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/ce224d9b10294a6cb295d550f2ec6bc8 2024-11-15T14:15:43,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/ef1a15fc765c4f09a1ff65ea6b5c42e8 is 50, key is test_row_0/B:col10/1731680143518/Put/seqid=0 2024-11-15T14:15:44,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T14:15:44,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:44,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742093_1269 (size=12151) 2024-11-15T14:15:44,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680204042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:44,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680204152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:44,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T14:15:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T14:15:44,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T14:15:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:44,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680204361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:44,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/ef1a15fc765c4f09a1ff65ea6b5c42e8 2024-11-15T14:15:44,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/59b40855d8f840a9be67fa84131f6421 is 50, key is test_row_0/C:col10/1731680143518/Put/seqid=0 2024-11-15T14:15:44,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T14:15:44,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:44,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T14:15:44,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742094_1270 (size=12151) 2024-11-15T14:15:44,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T14:15:44,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:44,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680204673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:44,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T14:15:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:44,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/59b40855d8f840a9be67fa84131f6421 2024-11-15T14:15:44,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/ce224d9b10294a6cb295d550f2ec6bc8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ce224d9b10294a6cb295d550f2ec6bc8 2024-11-15T14:15:44,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ce224d9b10294a6cb295d550f2ec6bc8, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T14:15:44,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/ef1a15fc765c4f09a1ff65ea6b5c42e8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/ef1a15fc765c4f09a1ff65ea6b5c42e8 2024-11-15T14:15:44,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/ef1a15fc765c4f09a1ff65ea6b5c42e8, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T14:15:44,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/59b40855d8f840a9be67fa84131f6421 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/59b40855d8f840a9be67fa84131f6421 2024-11-15T14:15:44,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/59b40855d8f840a9be67fa84131f6421, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T14:15:44,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e966eb2fe52084edc96ca33b9ecedaa2 in 1004ms, sequenceid=245, compaction requested=true 2024-11-15T14:15:44,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:44,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:44,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:44,921 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:44,921 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:44,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:44,949 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:44,949 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/A is initiating minor compaction (all files) 2024-11-15T14:15:44,949 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/A in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,949 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:44,950 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/B is initiating minor compaction (all files) 2024-11-15T14:15:44,950 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/82233b87b21f458d9abd963bb606c408, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/55814f091ad8412d90dcb06aad0fc6e4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ce224d9b10294a6cb295d550f2ec6bc8] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=36.0 K 2024-11-15T14:15:44,950 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/B in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,950 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/12f245dd7efe480eb3797cfac13fa320, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e2dceb32ff3844419881eed39dba2968, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/ef1a15fc765c4f09a1ff65ea6b5c42e8] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=36.0 K 2024-11-15T14:15:44,951 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82233b87b21f458d9abd963bb606c408, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731680142556 2024-11-15T14:15:44,951 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 12f245dd7efe480eb3797cfac13fa320, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731680142556 2024-11-15T14:15:44,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T14:15:44,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:44,951 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55814f091ad8412d90dcb06aad0fc6e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731680142834 2024-11-15T14:15:44,951 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T14:15:44,951 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting e2dceb32ff3844419881eed39dba2968, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731680142834 2024-11-15T14:15:44,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:44,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:44,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:44,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:44,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:44,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:44,952 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce224d9b10294a6cb295d550f2ec6bc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731680143518 2024-11-15T14:15:44,952 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting ef1a15fc765c4f09a1ff65ea6b5c42e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731680143518 2024-11-15T14:15:44,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:44,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:44,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:44,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:44,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:44,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/b73d1747880e417f8783196644959492 is 50, key is test_row_0/A:col10/1731680144949/Put/seqid=0 2024-11-15T14:15:44,973 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:44,973 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/0582cd1339f94a888259cecdf0803dce is 50, key is test_row_0/B:col10/1731680143518/Put/seqid=0 2024-11-15T14:15:44,975 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#A#compaction#228 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:44,976 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/45bf9b66da8e448c8e1e4d54ffff9a1e is 50, key is test_row_0/A:col10/1731680143518/Put/seqid=0 2024-11-15T14:15:44,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:44,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680204983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:44,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T14:15:44,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680204984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680204985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:44,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680204988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742095_1271 (size=12629) 2024-11-15T14:15:45,021 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/0582cd1339f94a888259cecdf0803dce as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/0582cd1339f94a888259cecdf0803dce 2024-11-15T14:15:45,029 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/B of e966eb2fe52084edc96ca33b9ecedaa2 into 0582cd1339f94a888259cecdf0803dce(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:45,032 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:45,032 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/B, priority=13, startTime=1731680144921; duration=0sec 2024-11-15T14:15:45,032 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:45,033 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:B 2024-11-15T14:15:45,033 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:45,034 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:45,034 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/C is initiating minor compaction (all files) 2024-11-15T14:15:45,034 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/C in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:45,034 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/9e6441ccbdd749f2af0b6cd41fbf8b6c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/2d5a69202a044c0d88e467a9eb5b6f4d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/59b40855d8f840a9be67fa84131f6421] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=36.0 K 2024-11-15T14:15:45,035 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e6441ccbdd749f2af0b6cd41fbf8b6c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731680142556 2024-11-15T14:15:45,035 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d5a69202a044c0d88e467a9eb5b6f4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731680142834 2024-11-15T14:15:45,035 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 59b40855d8f840a9be67fa84131f6421, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731680143518 2024-11-15T14:15:45,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742096_1272 (size=12629) 2024-11-15T14:15:45,052 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/45bf9b66da8e448c8e1e4d54ffff9a1e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/45bf9b66da8e448c8e1e4d54ffff9a1e 2024-11-15T14:15:45,054 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:45,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742097_1273 (size=17131) 2024-11-15T14:15:45,057 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/025c454535ef49818a25d6aecef710d8 is 50, key is test_row_0/C:col10/1731680143518/Put/seqid=0 2024-11-15T14:15:45,058 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/A of e966eb2fe52084edc96ca33b9ecedaa2 into 45bf9b66da8e448c8e1e4d54ffff9a1e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:45,058 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:45,059 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/A, priority=13, startTime=1731680144921; duration=0sec 2024-11-15T14:15:45,059 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:45,059 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:A 2024-11-15T14:15:45,059 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/b73d1747880e417f8783196644959492 2024-11-15T14:15:45,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742098_1274 (size=12629) 2024-11-15T14:15:45,086 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/025c454535ef49818a25d6aecef710d8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/025c454535ef49818a25d6aecef710d8 2024-11-15T14:15:45,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/e908977a1b7e4b17a90830b5cbdf970e is 50, key is test_row_0/B:col10/1731680144949/Put/seqid=0 2024-11-15T14:15:45,101 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/C of e966eb2fe52084edc96ca33b9ecedaa2 into 025c454535ef49818a25d6aecef710d8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:45,101 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:45,101 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/C, priority=13, startTime=1731680144955; duration=0sec 2024-11-15T14:15:45,101 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:45,101 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:C 2024-11-15T14:15:45,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680205102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680205102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680205102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680205102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742099_1275 (size=12301) 2024-11-15T14:15:45,127 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/e908977a1b7e4b17a90830b5cbdf970e 2024-11-15T14:15:45,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/3abc5f89cfe54347b152c9aaf6bf741b is 50, key is test_row_0/C:col10/1731680144949/Put/seqid=0 2024-11-15T14:15:45,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742100_1276 (size=12301) 2024-11-15T14:15:45,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680205182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,187 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/3abc5f89cfe54347b152c9aaf6bf741b 2024-11-15T14:15:45,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/b73d1747880e417f8783196644959492 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b73d1747880e417f8783196644959492 2024-11-15T14:15:45,204 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b73d1747880e417f8783196644959492, entries=250, sequenceid=266, filesize=16.7 K 2024-11-15T14:15:45,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/e908977a1b7e4b17a90830b5cbdf970e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e908977a1b7e4b17a90830b5cbdf970e 2024-11-15T14:15:45,210 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e908977a1b7e4b17a90830b5cbdf970e, entries=150, sequenceid=266, filesize=12.0 K 2024-11-15T14:15:45,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/3abc5f89cfe54347b152c9aaf6bf741b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/3abc5f89cfe54347b152c9aaf6bf741b 2024-11-15T14:15:45,219 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/3abc5f89cfe54347b152c9aaf6bf741b, entries=150, sequenceid=266, filesize=12.0 K 2024-11-15T14:15:45,220 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for e966eb2fe52084edc96ca33b9ecedaa2 in 269ms, sequenceid=266, compaction requested=false 2024-11-15T14:15:45,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:45,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:45,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-15T14:15:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-15T14:15:45,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-15T14:15:45,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3650 sec 2024-11-15T14:15:45,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.3730 sec 2024-11-15T14:15:45,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T14:15:45,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:45,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:45,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:45,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:45,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:45,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:45,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:45,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/24924703ee0749d48e2c679a5ba5a572 is 50, key is test_row_0/A:col10/1731680144984/Put/seqid=0 2024-11-15T14:15:45,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680205362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680205374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680205375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680205376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742101_1277 (size=12301) 2024-11-15T14:15:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/24924703ee0749d48e2c679a5ba5a572 2024-11-15T14:15:45,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/cea20b896490454d888eb959e51bb0bf is 50, key is test_row_0/B:col10/1731680144984/Put/seqid=0 2024-11-15T14:15:45,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742102_1278 (size=12301) 2024-11-15T14:15:45,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/cea20b896490454d888eb959e51bb0bf 2024-11-15T14:15:45,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/3e4ba79234bb4f0fa4b36d462eab425b is 50, key is test_row_0/C:col10/1731680144984/Put/seqid=0 2024-11-15T14:15:45,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680205482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680205492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680205492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680205492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742103_1279 (size=12301) 2024-11-15T14:15:45,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680205692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680205702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680205702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680205702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:45,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/3e4ba79234bb4f0fa4b36d462eab425b 2024-11-15T14:15:45,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/24924703ee0749d48e2c679a5ba5a572 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/24924703ee0749d48e2c679a5ba5a572 2024-11-15T14:15:45,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/24924703ee0749d48e2c679a5ba5a572, entries=150, sequenceid=282, filesize=12.0 K 2024-11-15T14:15:45,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/cea20b896490454d888eb959e51bb0bf as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/cea20b896490454d888eb959e51bb0bf 2024-11-15T14:15:45,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/cea20b896490454d888eb959e51bb0bf, entries=150, sequenceid=282, filesize=12.0 K 2024-11-15T14:15:45,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/3e4ba79234bb4f0fa4b36d462eab425b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/3e4ba79234bb4f0fa4b36d462eab425b 2024-11-15T14:15:45,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/3e4ba79234bb4f0fa4b36d462eab425b, entries=150, sequenceid=282, filesize=12.0 K 2024-11-15T14:15:45,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e966eb2fe52084edc96ca33b9ecedaa2 in 628ms, sequenceid=282, compaction requested=true 2024-11-15T14:15:45,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:45,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:45,944 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:45,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:45,944 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:45,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:45,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:45,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:45,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:45,945 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:45,945 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:45,945 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/A is initiating minor compaction (all files) 2024-11-15T14:15:45,945 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/B is initiating minor compaction (all files) 2024-11-15T14:15:45,945 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/A in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:45,945 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/B in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:45,946 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/0582cd1339f94a888259cecdf0803dce, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e908977a1b7e4b17a90830b5cbdf970e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/cea20b896490454d888eb959e51bb0bf] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=36.4 K 2024-11-15T14:15:45,946 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/45bf9b66da8e448c8e1e4d54ffff9a1e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b73d1747880e417f8783196644959492, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/24924703ee0749d48e2c679a5ba5a572] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=41.1 K 2024-11-15T14:15:45,946 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0582cd1339f94a888259cecdf0803dce, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731680143518 2024-11-15T14:15:45,946 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45bf9b66da8e448c8e1e4d54ffff9a1e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731680143518 2024-11-15T14:15:45,946 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting e908977a1b7e4b17a90830b5cbdf970e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731680144027 2024-11-15T14:15:45,946 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b73d1747880e417f8783196644959492, keycount=250, bloomtype=ROW, size=16.7 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731680143965 2024-11-15T14:15:45,951 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting cea20b896490454d888eb959e51bb0bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731680144984 2024-11-15T14:15:45,952 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24924703ee0749d48e2c679a5ba5a572, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731680144984 2024-11-15T14:15:45,979 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#A#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:45,979 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/216aefb8bdd94d6a8656df61d5ae99c4 is 50, key is test_row_0/A:col10/1731680144984/Put/seqid=0 2024-11-15T14:15:45,991 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#B#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:45,992 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/fc7888732d814f8eb9a6a4ee51903096 is 50, key is test_row_0/B:col10/1731680144984/Put/seqid=0 2024-11-15T14:15:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:46,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T14:15:46,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:46,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:46,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:46,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:46,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:46,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T14:15:46,012 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:46,013 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-15T14:15:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T14:15:46,016 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:46,018 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:46,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:46,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742104_1280 (size=12881) 2024-11-15T14:15:46,033 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/216aefb8bdd94d6a8656df61d5ae99c4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/216aefb8bdd94d6a8656df61d5ae99c4 2024-11-15T14:15:46,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680206030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680206031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680206041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680206042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742105_1281 (size=12881) 2024-11-15T14:15:46,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/9b2db47bb5db41c796bb8e9c43e18fc5 is 50, key is test_row_0/A:col10/1731680145367/Put/seqid=0 2024-11-15T14:15:46,054 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/fc7888732d814f8eb9a6a4ee51903096 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/fc7888732d814f8eb9a6a4ee51903096 2024-11-15T14:15:46,057 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/A of e966eb2fe52084edc96ca33b9ecedaa2 into 216aefb8bdd94d6a8656df61d5ae99c4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:46,057 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:46,057 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/A, priority=13, startTime=1731680145944; duration=0sec 2024-11-15T14:15:46,060 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:46,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742106_1282 (size=14741) 2024-11-15T14:15:46,062 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:A 2024-11-15T14:15:46,062 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:46,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/9b2db47bb5db41c796bb8e9c43e18fc5 2024-11-15T14:15:46,063 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/B of e966eb2fe52084edc96ca33b9ecedaa2 into fc7888732d814f8eb9a6a4ee51903096(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:46,063 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:46,064 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/B, priority=13, startTime=1731680145944; duration=0sec 2024-11-15T14:15:46,064 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:46,064 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:B 2024-11-15T14:15:46,066 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:46,066 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/C is initiating minor compaction (all files) 2024-11-15T14:15:46,066 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/C in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:46,066 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/025c454535ef49818a25d6aecef710d8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/3abc5f89cfe54347b152c9aaf6bf741b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/3e4ba79234bb4f0fa4b36d462eab425b] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=36.4 K 2024-11-15T14:15:46,067 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 025c454535ef49818a25d6aecef710d8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731680143518 2024-11-15T14:15:46,067 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3abc5f89cfe54347b152c9aaf6bf741b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731680144027 2024-11-15T14:15:46,068 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e4ba79234bb4f0fa4b36d462eab425b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731680144984 2024-11-15T14:15:46,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/c029a002362547e6893359b173aa4f29 is 50, key is test_row_0/B:col10/1731680145367/Put/seqid=0 2024-11-15T14:15:46,091 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#C#compaction#240 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:46,093 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ec10bc6981844030b3356a236c751656 is 50, key is test_row_0/C:col10/1731680144984/Put/seqid=0 2024-11-15T14:15:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T14:15:46,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742107_1283 (size=12301) 2024-11-15T14:15:46,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/c029a002362547e6893359b173aa4f29 2024-11-15T14:15:46,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680206142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680206152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680206152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680206152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742108_1284 (size=12881) 2024-11-15T14:15:46,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T14:15:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:46,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,178 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ec10bc6981844030b3356a236c751656 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ec10bc6981844030b3356a236c751656 2024-11-15T14:15:46,185 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/C of e966eb2fe52084edc96ca33b9ecedaa2 into ec10bc6981844030b3356a236c751656(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:46,186 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:46,186 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/C, priority=13, startTime=1731680145944; duration=0sec 2024-11-15T14:15:46,186 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:46,186 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:C 2024-11-15T14:15:46,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/e20791e39ae24de5b3a003ea63ca54c2 is 50, key is test_row_0/C:col10/1731680145367/Put/seqid=0 2024-11-15T14:15:46,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680206202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,209 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in e966eb2fe52084edc96ca33b9ecedaa2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742109_1285 (size=12301) 2024-11-15T14:15:46,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T14:15:46,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:46,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:46,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:46,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T14:15:46,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680206352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680206362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680206362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680206362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T14:15:46,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:46,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:46,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:46,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/e20791e39ae24de5b3a003ea63ca54c2 2024-11-15T14:15:46,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/9b2db47bb5db41c796bb8e9c43e18fc5 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/9b2db47bb5db41c796bb8e9c43e18fc5 2024-11-15T14:15:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T14:15:46,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/9b2db47bb5db41c796bb8e9c43e18fc5, entries=200, sequenceid=302, filesize=14.4 K 2024-11-15T14:15:46,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/c029a002362547e6893359b173aa4f29 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/c029a002362547e6893359b173aa4f29 2024-11-15T14:15:46,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/c029a002362547e6893359b173aa4f29, entries=150, sequenceid=302, filesize=12.0 K 2024-11-15T14:15:46,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/e20791e39ae24de5b3a003ea63ca54c2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/e20791e39ae24de5b3a003ea63ca54c2 2024-11-15T14:15:46,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/e20791e39ae24de5b3a003ea63ca54c2, entries=150, sequenceid=302, filesize=12.0 K 2024-11-15T14:15:46,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e966eb2fe52084edc96ca33b9ecedaa2 in 653ms, sequenceid=302, compaction requested=false 2024-11-15T14:15:46,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:46,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T14:15:46,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:46,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:46,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:46,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:46,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:46,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:46,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T14:15:46,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:46,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:46,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:46,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/9fd42392e8744fa38cb9c30b8e134a64 is 50, key is test_row_0/A:col10/1731680146034/Put/seqid=0 2024-11-15T14:15:46,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680206699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680206707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680206709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742110_1286 (size=14741) 2024-11-15T14:15:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680206706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/9fd42392e8744fa38cb9c30b8e134a64 2024-11-15T14:15:46,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/3ef2999b01fd47be84d81a509fef9e2a is 50, key is test_row_0/B:col10/1731680146034/Put/seqid=0 2024-11-15T14:15:46,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742111_1287 (size=12301) 2024-11-15T14:15:46,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/3ef2999b01fd47be84d81a509fef9e2a 2024-11-15T14:15:46,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ebcdac0058324f08927938f176806aaa is 50, key is test_row_0/C:col10/1731680146034/Put/seqid=0 2024-11-15T14:15:46,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680206812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680206821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680206821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:46,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680206821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:46,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T14:15:46,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:46,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:46,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742112_1288 (size=12301) 2024-11-15T14:15:46,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T14:15:46,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:46,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:46,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:46,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:47,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680207023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680207032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680207032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680207032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T14:15:47,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:47,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:47,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:47,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:47,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T14:15:47,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ebcdac0058324f08927938f176806aaa 2024-11-15T14:15:47,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/9fd42392e8744fa38cb9c30b8e134a64 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/9fd42392e8744fa38cb9c30b8e134a64 2024-11-15T14:15:47,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/9fd42392e8744fa38cb9c30b8e134a64, entries=200, sequenceid=318, filesize=14.4 K 2024-11-15T14:15:47,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/3ef2999b01fd47be84d81a509fef9e2a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/3ef2999b01fd47be84d81a509fef9e2a 2024-11-15T14:15:47,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T14:15:47,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:47,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:47,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:47,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:47,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:47,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/3ef2999b01fd47be84d81a509fef9e2a, entries=150, sequenceid=318, filesize=12.0 K 2024-11-15T14:15:47,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ebcdac0058324f08927938f176806aaa as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ebcdac0058324f08927938f176806aaa 2024-11-15T14:15:47,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ebcdac0058324f08927938f176806aaa, entries=150, sequenceid=318, filesize=12.0 K 2024-11-15T14:15:47,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e966eb2fe52084edc96ca33b9ecedaa2 in 637ms, sequenceid=318, compaction requested=true 2024-11-15T14:15:47,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:47,305 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:47,309 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:47,309 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/A is initiating minor compaction (all files) 2024-11-15T14:15:47,309 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/A in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:47,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:47,310 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/216aefb8bdd94d6a8656df61d5ae99c4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/9b2db47bb5db41c796bb8e9c43e18fc5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/9fd42392e8744fa38cb9c30b8e134a64] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=41.4 K 2024-11-15T14:15:47,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:47,310 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:47,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:47,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:47,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:47,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:47,310 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 216aefb8bdd94d6a8656df61d5ae99c4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731680144984 2024-11-15T14:15:47,312 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b2db47bb5db41c796bb8e9c43e18fc5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731680145367 2024-11-15T14:15:47,313 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:47,313 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/B is initiating minor compaction (all files) 2024-11-15T14:15:47,313 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/B in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:47,314 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/fc7888732d814f8eb9a6a4ee51903096, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/c029a002362547e6893359b173aa4f29, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/3ef2999b01fd47be84d81a509fef9e2a] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=36.6 K 2024-11-15T14:15:47,314 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fd42392e8744fa38cb9c30b8e134a64, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731680146020 2024-11-15T14:15:47,314 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting fc7888732d814f8eb9a6a4ee51903096, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731680144984 2024-11-15T14:15:47,315 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c029a002362547e6893359b173aa4f29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731680145367 2024-11-15T14:15:47,315 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ef2999b01fd47be84d81a509fef9e2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731680146020 2024-11-15T14:15:47,341 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#A#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:47,342 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/0ffa71429fb24283916a56b7c71014ab is 50, key is test_row_0/A:col10/1731680146034/Put/seqid=0 2024-11-15T14:15:47,345 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#B#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:47,345 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/1e4bfb5925074df79c10037f14e91eaa is 50, key is test_row_0/B:col10/1731680146034/Put/seqid=0 2024-11-15T14:15:47,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T14:15:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:47,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742113_1289 (size=12983) 2024-11-15T14:15:47,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680207370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680207375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680207383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680207384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,400 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/0ffa71429fb24283916a56b7c71014ab as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0ffa71429fb24283916a56b7c71014ab 2024-11-15T14:15:47,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/67ed6d66a0b44416b2a46d6000b49197 is 50, key is test_row_0/A:col10/1731680146707/Put/seqid=0 2024-11-15T14:15:47,415 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/A of e966eb2fe52084edc96ca33b9ecedaa2 into 0ffa71429fb24283916a56b7c71014ab(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:47,415 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:47,415 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/A, priority=13, startTime=1731680147305; duration=0sec 2024-11-15T14:15:47,415 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:47,415 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:A 2024-11-15T14:15:47,415 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:47,418 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:47,418 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/C is initiating minor compaction (all files) 2024-11-15T14:15:47,418 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/C in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:47,418 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ec10bc6981844030b3356a236c751656, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/e20791e39ae24de5b3a003ea63ca54c2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ebcdac0058324f08927938f176806aaa] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=36.6 K 2024-11-15T14:15:47,418 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec10bc6981844030b3356a236c751656, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731680144984 2024-11-15T14:15:47,419 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e20791e39ae24de5b3a003ea63ca54c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731680145367 2024-11-15T14:15:47,419 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebcdac0058324f08927938f176806aaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731680146020 2024-11-15T14:15:47,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742114_1290 (size=12983) 2024-11-15T14:15:47,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T14:15:47,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:47,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:47,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:47,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:47,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:47,466 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:47,466 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/fda15d27d9a94ac6a93ab12eae6d9c5b is 50, key is test_row_0/C:col10/1731680146034/Put/seqid=0 2024-11-15T14:15:47,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742115_1291 (size=14741) 2024-11-15T14:15:47,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/67ed6d66a0b44416b2a46d6000b49197 2024-11-15T14:15:47,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680207492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680207492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680207502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680207502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/089b97e266984f2683bd6f86e6a6b7d4 is 50, key is test_row_0/B:col10/1731680146707/Put/seqid=0 2024-11-15T14:15:47,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742116_1292 (size=12983) 2024-11-15T14:15:47,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742117_1293 (size=12301) 2024-11-15T14:15:47,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/089b97e266984f2683bd6f86e6a6b7d4 2024-11-15T14:15:47,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T14:15:47,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:47,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:47,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:47,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:47,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:47,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/8cbe7c236e62435d9cdefb42f74e0783 is 50, key is test_row_0/C:col10/1731680146707/Put/seqid=0 2024-11-15T14:15:47,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742118_1294 (size=12301) 2024-11-15T14:15:47,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/8cbe7c236e62435d9cdefb42f74e0783 2024-11-15T14:15:47,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680207702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680207702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/67ed6d66a0b44416b2a46d6000b49197 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/67ed6d66a0b44416b2a46d6000b49197 2024-11-15T14:15:47,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/67ed6d66a0b44416b2a46d6000b49197, entries=200, sequenceid=338, filesize=14.4 K 2024-11-15T14:15:47,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/089b97e266984f2683bd6f86e6a6b7d4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/089b97e266984f2683bd6f86e6a6b7d4 2024-11-15T14:15:47,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680207723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680207723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:47,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/089b97e266984f2683bd6f86e6a6b7d4, entries=150, sequenceid=338, filesize=12.0 K 2024-11-15T14:15:47,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/8cbe7c236e62435d9cdefb42f74e0783 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/8cbe7c236e62435d9cdefb42f74e0783 2024-11-15T14:15:47,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/8cbe7c236e62435d9cdefb42f74e0783, entries=150, sequenceid=338, filesize=12.0 K 2024-11-15T14:15:47,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e966eb2fe52084edc96ca33b9ecedaa2 in 392ms, sequenceid=338, compaction requested=false 2024-11-15T14:15:47,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:47,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T14:15:47,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:47,764 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T14:15:47,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:47,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:47,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:47,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:47,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:47,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:47,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/ee66cc9a317f452cb18613c021b4a341 is 50, key is test_row_0/A:col10/1731680147362/Put/seqid=0 2024-11-15T14:15:47,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742119_1295 (size=12301) 2024-11-15T14:15:47,842 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/1e4bfb5925074df79c10037f14e91eaa as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/1e4bfb5925074df79c10037f14e91eaa 2024-11-15T14:15:47,849 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/B of e966eb2fe52084edc96ca33b9ecedaa2 into 1e4bfb5925074df79c10037f14e91eaa(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:47,849 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:47,849 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/B, priority=13, startTime=1731680147310; duration=0sec 2024-11-15T14:15:47,859 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:47,859 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:B 2024-11-15T14:15:47,947 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/fda15d27d9a94ac6a93ab12eae6d9c5b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/fda15d27d9a94ac6a93ab12eae6d9c5b 2024-11-15T14:15:47,957 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/C of e966eb2fe52084edc96ca33b9ecedaa2 into fda15d27d9a94ac6a93ab12eae6d9c5b(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:47,957 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:47,957 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/C, priority=13, startTime=1731680147310; duration=0sec 2024-11-15T14:15:47,958 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:47,958 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:C 2024-11-15T14:15:48,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:48,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680208080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680208082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680208083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680208084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T14:15:48,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680208202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680208202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680208202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680208202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,231 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/ee66cc9a317f452cb18613c021b4a341 2024-11-15T14:15:48,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680208231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,239 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in e966eb2fe52084edc96ca33b9ecedaa2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/c5115c1c20c94077a7f0d57097e6d72e is 50, key is test_row_0/B:col10/1731680147362/Put/seqid=0 2024-11-15T14:15:48,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742120_1296 (size=12301) 2024-11-15T14:15:48,284 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/c5115c1c20c94077a7f0d57097e6d72e 2024-11-15T14:15:48,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/97e9b1604e3e4bc19bf3395073384072 is 50, key is test_row_0/C:col10/1731680147362/Put/seqid=0 2024-11-15T14:15:48,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742121_1297 (size=12301) 2024-11-15T14:15:48,324 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/97e9b1604e3e4bc19bf3395073384072 2024-11-15T14:15:48,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/ee66cc9a317f452cb18613c021b4a341 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ee66cc9a317f452cb18613c021b4a341 2024-11-15T14:15:48,341 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ee66cc9a317f452cb18613c021b4a341, entries=150, sequenceid=351, filesize=12.0 K 2024-11-15T14:15:48,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/c5115c1c20c94077a7f0d57097e6d72e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/c5115c1c20c94077a7f0d57097e6d72e 2024-11-15T14:15:48,351 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/c5115c1c20c94077a7f0d57097e6d72e, entries=150, sequenceid=351, filesize=12.0 K 2024-11-15T14:15:48,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/97e9b1604e3e4bc19bf3395073384072 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/97e9b1604e3e4bc19bf3395073384072 2024-11-15T14:15:48,364 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/97e9b1604e3e4bc19bf3395073384072, entries=150, sequenceid=351, filesize=12.0 K 2024-11-15T14:15:48,367 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e966eb2fe52084edc96ca33b9ecedaa2 in 603ms, sequenceid=351, compaction requested=true 2024-11-15T14:15:48,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:48,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:48,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-15T14:15:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-15T14:15:48,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-15T14:15:48,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3510 sec 2024-11-15T14:15:48,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.3640 sec 2024-11-15T14:15:48,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T14:15:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:48,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731680148078, max timestamp=1731680148416Num uniques 859; 2024-11-15T14:15:48,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7050, suffix heap size=117064,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=154 2024-11-15T14:15:48,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:48,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:48,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:48,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:48,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:48,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:48,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/34b71eb89cc840989fafaf10109f51eb is 50, key is test_row_0/A:col10/1731680148416/Put/seqid=0 2024-11-15T14:15:48,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680208456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680208461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680208464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742122_1298 (size=17181) 2024-11-15T14:15:48,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680208466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680208572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680208572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680208581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680208581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680208782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680208782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680208791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680208793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:48,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.20 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/34b71eb89cc840989fafaf10109f51eb 2024-11-15T14:15:48,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/9da440f8472b49c4b4380800fa2c4cc1 is 50, key is test_row_0/B:col10/1731680148416/Put/seqid=0 2024-11-15T14:15:48,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742123_1299 (size=12301) 2024-11-15T14:15:48,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/9da440f8472b49c4b4380800fa2c4cc1 2024-11-15T14:15:48,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/fdc1200dc88b409287beac962d3d1dd4 is 50, key is test_row_0/C:col10/1731680148416/Put/seqid=0 2024-11-15T14:15:49,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742124_1300 (size=12301) 2024-11-15T14:15:49,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680209092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680209092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:49,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680209101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:49,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680209101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:49,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/fdc1200dc88b409287beac962d3d1dd4 2024-11-15T14:15:49,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/34b71eb89cc840989fafaf10109f51eb as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/34b71eb89cc840989fafaf10109f51eb 2024-11-15T14:15:49,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/34b71eb89cc840989fafaf10109f51eb, entries=250, sequenceid=376, filesize=16.8 K 2024-11-15T14:15:49,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/9da440f8472b49c4b4380800fa2c4cc1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/9da440f8472b49c4b4380800fa2c4cc1 2024-11-15T14:15:49,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/9da440f8472b49c4b4380800fa2c4cc1, entries=150, sequenceid=376, filesize=12.0 K 2024-11-15T14:15:49,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/fdc1200dc88b409287beac962d3d1dd4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/fdc1200dc88b409287beac962d3d1dd4 2024-11-15T14:15:49,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/fdc1200dc88b409287beac962d3d1dd4, entries=150, sequenceid=376, filesize=12.0 K 2024-11-15T14:15:49,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.65 KB/105115, heapSize ~301.40 KB/308632, currentSize=73.80 KB/75570 for e966eb2fe52084edc96ca33b9ecedaa2 in 1053ms, sequenceid=376, compaction requested=true 2024-11-15T14:15:49,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:49,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:49,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:49,472 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:49,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:49,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:49,472 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:49,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:49,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:49,474 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:49,474 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/B is initiating minor compaction (all files) 2024-11-15T14:15:49,474 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/B in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:49,474 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/1e4bfb5925074df79c10037f14e91eaa, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/089b97e266984f2683bd6f86e6a6b7d4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/c5115c1c20c94077a7f0d57097e6d72e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/9da440f8472b49c4b4380800fa2c4cc1] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=48.7 K 2024-11-15T14:15:49,475 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e4bfb5925074df79c10037f14e91eaa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731680146020 2024-11-15T14:15:49,475 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57206 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:49,475 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/A is initiating minor compaction (all files) 2024-11-15T14:15:49,475 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/A in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:49,475 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0ffa71429fb24283916a56b7c71014ab, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/67ed6d66a0b44416b2a46d6000b49197, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ee66cc9a317f452cb18613c021b4a341, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/34b71eb89cc840989fafaf10109f51eb] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=55.9 K 2024-11-15T14:15:49,476 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 089b97e266984f2683bd6f86e6a6b7d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731680146688 2024-11-15T14:15:49,476 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ffa71429fb24283916a56b7c71014ab, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731680146020 2024-11-15T14:15:49,476 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67ed6d66a0b44416b2a46d6000b49197, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731680146688 2024-11-15T14:15:49,476 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c5115c1c20c94077a7f0d57097e6d72e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731680147362 2024-11-15T14:15:49,477 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee66cc9a317f452cb18613c021b4a341, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731680147362 2024-11-15T14:15:49,477 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9da440f8472b49c4b4380800fa2c4cc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731680148078 2024-11-15T14:15:49,478 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34b71eb89cc840989fafaf10109f51eb, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731680148078 2024-11-15T14:15:49,491 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#B#compaction#257 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:49,492 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/61afd9f2cf99461f90baee9fa2593344 is 50, key is test_row_0/B:col10/1731680148416/Put/seqid=0 2024-11-15T14:15:49,510 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#A#compaction#258 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:49,511 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/b1bd4310c1464bdc8752fcedc3e4da0c is 50, key is test_row_0/A:col10/1731680148416/Put/seqid=0 2024-11-15T14:15:49,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742125_1301 (size=13119) 2024-11-15T14:15:49,561 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/61afd9f2cf99461f90baee9fa2593344 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/61afd9f2cf99461f90baee9fa2593344 2024-11-15T14:15:49,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742126_1302 (size=13119) 2024-11-15T14:15:49,585 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/B of e966eb2fe52084edc96ca33b9ecedaa2 into 61afd9f2cf99461f90baee9fa2593344(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:49,585 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:49,585 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/B, priority=12, startTime=1731680149472; duration=0sec 2024-11-15T14:15:49,586 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:49,586 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:B 2024-11-15T14:15:49,586 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:49,588 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:49,588 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/C is initiating minor compaction (all files) 2024-11-15T14:15:49,589 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/C in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:49,589 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/fda15d27d9a94ac6a93ab12eae6d9c5b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/8cbe7c236e62435d9cdefb42f74e0783, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/97e9b1604e3e4bc19bf3395073384072, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/fdc1200dc88b409287beac962d3d1dd4] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=48.7 K 2024-11-15T14:15:49,589 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting fda15d27d9a94ac6a93ab12eae6d9c5b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731680146020 2024-11-15T14:15:49,590 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cbe7c236e62435d9cdefb42f74e0783, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731680146688 2024-11-15T14:15:49,590 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 97e9b1604e3e4bc19bf3395073384072, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731680147362 2024-11-15T14:15:49,590 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting fdc1200dc88b409287beac962d3d1dd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731680148078 2024-11-15T14:15:49,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T14:15:49,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:49,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:49,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:49,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:49,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:49,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:49,621 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#C#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:49,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/439014429113463ba51ea5fbce13929b is 50, key is test_row_0/A:col10/1731680149608/Put/seqid=0 2024-11-15T14:15:49,626 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/a89d500d910d402b82469936ce7e2718 is 50, key is test_row_0/C:col10/1731680148416/Put/seqid=0 2024-11-15T14:15:49,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742127_1303 (size=14741) 2024-11-15T14:15:49,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742128_1304 (size=13119) 2024-11-15T14:15:49,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680209681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:49,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:49,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680209693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:49,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:49,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680209696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:49,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680209698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:49,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680209801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:49,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680209811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:49,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680209812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:49,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680209812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:49,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:49,983 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/b1bd4310c1464bdc8752fcedc3e4da0c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b1bd4310c1464bdc8752fcedc3e4da0c 2024-11-15T14:15:49,992 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/A of e966eb2fe52084edc96ca33b9ecedaa2 into b1bd4310c1464bdc8752fcedc3e4da0c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:49,992 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:49,992 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/A, priority=12, startTime=1731680149472; duration=0sec 2024-11-15T14:15:49,992 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:49,993 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:A 2024-11-15T14:15:50,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680210013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:50,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:50,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680210031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:50,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680210031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:50,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:50,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680210034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:50,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/439014429113463ba51ea5fbce13929b 2024-11-15T14:15:50,098 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/a89d500d910d402b82469936ce7e2718 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/a89d500d910d402b82469936ce7e2718 2024-11-15T14:15:50,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/4741b21f82da4ba08dc43ad4cbe60b69 is 50, key is test_row_0/B:col10/1731680149608/Put/seqid=0 2024-11-15T14:15:50,109 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/C of e966eb2fe52084edc96ca33b9ecedaa2 into a89d500d910d402b82469936ce7e2718(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:50,109 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:50,109 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/C, priority=12, startTime=1731680149472; duration=0sec 2024-11-15T14:15:50,109 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:50,109 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:C 2024-11-15T14:15:50,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742129_1305 (size=12301) 2024-11-15T14:15:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T14:15:50,172 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:50,173 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-15T14:15:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T14:15:50,176 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:50,179 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:50,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T14:15:50,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T14:15:50,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:50,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:50,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:50,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:50,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:50,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:50,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:50,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680210335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680210342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680210342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:50,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680210352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:50,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T14:15:50,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:50,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:50,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:50,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:50,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T14:15:50,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/4741b21f82da4ba08dc43ad4cbe60b69 2024-11-15T14:15:50,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/0eb2ac2902034a24b73a9a75a3c6384f is 50, key is test_row_0/C:col10/1731680149608/Put/seqid=0 2024-11-15T14:15:50,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742130_1306 (size=12301) 2024-11-15T14:15:50,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/0eb2ac2902034a24b73a9a75a3c6384f 2024-11-15T14:15:50,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/439014429113463ba51ea5fbce13929b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/439014429113463ba51ea5fbce13929b 2024-11-15T14:15:50,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/439014429113463ba51ea5fbce13929b, entries=200, sequenceid=392, filesize=14.4 K 2024-11-15T14:15:50,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/4741b21f82da4ba08dc43ad4cbe60b69 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/4741b21f82da4ba08dc43ad4cbe60b69 2024-11-15T14:15:50,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/4741b21f82da4ba08dc43ad4cbe60b69, entries=150, sequenceid=392, filesize=12.0 K 2024-11-15T14:15:50,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/0eb2ac2902034a24b73a9a75a3c6384f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/0eb2ac2902034a24b73a9a75a3c6384f 2024-11-15T14:15:50,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/0eb2ac2902034a24b73a9a75a3c6384f, entries=150, sequenceid=392, filesize=12.0 K 2024-11-15T14:15:50,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e966eb2fe52084edc96ca33b9ecedaa2 in 1028ms, sequenceid=392, compaction requested=false 2024-11-15T14:15:50,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:50,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T14:15:50,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:50,644 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T14:15:50,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:50,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:50,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:50,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:50,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:50,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:50,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/d9d819a2c910434a9015c75529992d0c is 50, key is test_row_0/A:col10/1731680149695/Put/seqid=0 2024-11-15T14:15:50,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742131_1307 (size=12301) 2024-11-15T14:15:50,705 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/d9d819a2c910434a9015c75529992d0c 2024-11-15T14:15:50,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/2666b103d0764c5a90a96fc64adb5c6a is 50, key is test_row_0/B:col10/1731680149695/Put/seqid=0 2024-11-15T14:15:50,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742132_1308 (size=12301) 2024-11-15T14:15:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T14:15:50,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:50,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680210890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:50,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680210895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680210900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:50,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680210900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680211012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680211023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680211023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680211025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,176 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/2666b103d0764c5a90a96fc64adb5c6a 2024-11-15T14:15:51,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/00868a69fd644e16bf57c7a579032a66 is 50, key is test_row_0/C:col10/1731680149695/Put/seqid=0 2024-11-15T14:15:51,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680211221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680211232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680211233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680211232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742133_1309 (size=12301) 2024-11-15T14:15:51,258 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/00868a69fd644e16bf57c7a579032a66 2024-11-15T14:15:51,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/d9d819a2c910434a9015c75529992d0c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/d9d819a2c910434a9015c75529992d0c 2024-11-15T14:15:51,295 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/d9d819a2c910434a9015c75529992d0c, entries=150, sequenceid=411, filesize=12.0 K 2024-11-15T14:15:51,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/2666b103d0764c5a90a96fc64adb5c6a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/2666b103d0764c5a90a96fc64adb5c6a 2024-11-15T14:15:51,305 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/2666b103d0764c5a90a96fc64adb5c6a, entries=150, sequenceid=411, filesize=12.0 K 2024-11-15T14:15:51,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/00868a69fd644e16bf57c7a579032a66 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/00868a69fd644e16bf57c7a579032a66 2024-11-15T14:15:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T14:15:51,317 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/00868a69fd644e16bf57c7a579032a66, entries=150, sequenceid=411, filesize=12.0 K 2024-11-15T14:15:51,319 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for e966eb2fe52084edc96ca33b9ecedaa2 in 676ms, sequenceid=411, compaction requested=true 2024-11-15T14:15:51,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:51,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:51,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-15T14:15:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-15T14:15:51,333 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-15T14:15:51,333 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1500 sec 2024-11-15T14:15:51,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.1620 sec 2024-11-15T14:15:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:51,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T14:15:51,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:51,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:51,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:51,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:51,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:51,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:51,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/146023d470834305955575b50f567d51 is 50, key is test_row_0/A:col10/1731680150870/Put/seqid=0 2024-11-15T14:15:51,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742134_1310 (size=12301) 2024-11-15T14:15:51,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680211579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680211582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680211583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680211592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680211705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680211705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680211705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680211713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680211912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680211912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680211912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680211932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:51,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:51,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/146023d470834305955575b50f567d51 2024-11-15T14:15:52,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/d3bd1cda8ddf4390b35f2186a87e54c2 is 50, key is test_row_0/B:col10/1731680150870/Put/seqid=0 2024-11-15T14:15:52,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742135_1311 (size=12301) 2024-11-15T14:15:52,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680212221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:52,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680212232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680212232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:52,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680212242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:52,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:52,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60960 deadline: 1731680212272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:52,275 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in e966eb2fe52084edc96ca33b9ecedaa2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:15:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T14:15:52,334 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:52,335 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-15T14:15:52,338 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T14:15:52,339 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:52,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:52,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/d3bd1cda8ddf4390b35f2186a87e54c2 2024-11-15T14:15:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T14:15:52,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/7911b2ee86334be0a482a5aa7651122a is 50, key is test_row_0/C:col10/1731680150870/Put/seqid=0 2024-11-15T14:15:52,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T14:15:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:52,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742136_1312 (size=12301) 2024-11-15T14:15:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:52,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:52,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T14:15:52,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T14:15:52,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:52,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:52,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:52,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:52,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:52,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680212732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:52,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:52,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680212741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:52,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680212741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:52,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680212752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:52,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T14:15:52,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:52,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:52,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:52,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:52,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:52,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/7911b2ee86334be0a482a5aa7651122a 2024-11-15T14:15:52,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/146023d470834305955575b50f567d51 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/146023d470834305955575b50f567d51 2024-11-15T14:15:52,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/146023d470834305955575b50f567d51, entries=150, sequenceid=429, filesize=12.0 K 2024-11-15T14:15:52,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/d3bd1cda8ddf4390b35f2186a87e54c2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/d3bd1cda8ddf4390b35f2186a87e54c2 2024-11-15T14:15:52,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/d3bd1cda8ddf4390b35f2186a87e54c2, entries=150, sequenceid=429, filesize=12.0 K 2024-11-15T14:15:52,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/7911b2ee86334be0a482a5aa7651122a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/7911b2ee86334be0a482a5aa7651122a 2024-11-15T14:15:52,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/7911b2ee86334be0a482a5aa7651122a, entries=150, sequenceid=429, filesize=12.0 K 2024-11-15T14:15:52,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e966eb2fe52084edc96ca33b9ecedaa2 in 1389ms, sequenceid=429, compaction requested=true 2024-11-15T14:15:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:52,928 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:15:52,928 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:52,929 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:52,930 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/B is initiating minor compaction (all files) 2024-11-15T14:15:52,930 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/B in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:52,930 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52462 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:52,930 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/61afd9f2cf99461f90baee9fa2593344, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/4741b21f82da4ba08dc43ad4cbe60b69, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/2666b103d0764c5a90a96fc64adb5c6a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/d3bd1cda8ddf4390b35f2186a87e54c2] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=48.8 K 2024-11-15T14:15:52,930 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/A is initiating minor compaction (all files) 2024-11-15T14:15:52,930 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/A in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:52,930 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b1bd4310c1464bdc8752fcedc3e4da0c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/439014429113463ba51ea5fbce13929b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/d9d819a2c910434a9015c75529992d0c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/146023d470834305955575b50f567d51] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=51.2 K 2024-11-15T14:15:52,930 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1bd4310c1464bdc8752fcedc3e4da0c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731680148078 2024-11-15T14:15:52,931 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 439014429113463ba51ea5fbce13929b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731680148459 2024-11-15T14:15:52,931 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 61afd9f2cf99461f90baee9fa2593344, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731680148078 2024-11-15T14:15:52,931 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9d819a2c910434a9015c75529992d0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731680149670 2024-11-15T14:15:52,931 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4741b21f82da4ba08dc43ad4cbe60b69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731680148461 2024-11-15T14:15:52,931 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 146023d470834305955575b50f567d51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731680150870 2024-11-15T14:15:52,932 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2666b103d0764c5a90a96fc64adb5c6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731680149670 2024-11-15T14:15:52,932 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting d3bd1cda8ddf4390b35f2186a87e54c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731680150870 2024-11-15T14:15:52,946 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#A#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:52,946 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/f2ab4586d51c4755adb9ba952cad98c2 is 50, key is test_row_0/A:col10/1731680150870/Put/seqid=0 2024-11-15T14:15:52,957 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#B#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:52,957 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/06886368f70c4c1e9260192a0e0828da is 50, key is test_row_0/B:col10/1731680150870/Put/seqid=0 2024-11-15T14:15:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T14:15:52,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T14:15:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:52,964 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T14:15:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:52,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742137_1313 (size=13255) 2024-11-15T14:15:53,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/bf3d027153b74308837056eef9ca0b17 is 50, key is test_row_0/A:col10/1731680151564/Put/seqid=0 2024-11-15T14:15:53,006 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/f2ab4586d51c4755adb9ba952cad98c2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f2ab4586d51c4755adb9ba952cad98c2 2024-11-15T14:15:53,015 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/A of e966eb2fe52084edc96ca33b9ecedaa2 into f2ab4586d51c4755adb9ba952cad98c2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:53,015 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:53,015 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/A, priority=12, startTime=1731680152928; duration=0sec 2024-11-15T14:15:53,015 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:53,015 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:A 2024-11-15T14:15:53,015 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:15:53,017 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:15:53,018 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/C is initiating minor compaction (all files) 2024-11-15T14:15:53,018 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/C in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:53,018 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/a89d500d910d402b82469936ce7e2718, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/0eb2ac2902034a24b73a9a75a3c6384f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/00868a69fd644e16bf57c7a579032a66, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/7911b2ee86334be0a482a5aa7651122a] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=48.8 K 2024-11-15T14:15:53,018 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a89d500d910d402b82469936ce7e2718, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731680148078 2024-11-15T14:15:53,019 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eb2ac2902034a24b73a9a75a3c6384f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731680148461 2024-11-15T14:15:53,019 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00868a69fd644e16bf57c7a579032a66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731680149670 2024-11-15T14:15:53,019 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7911b2ee86334be0a482a5aa7651122a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731680150870 2024-11-15T14:15:53,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742138_1314 (size=13255) 2024-11-15T14:15:53,027 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/06886368f70c4c1e9260192a0e0828da as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/06886368f70c4c1e9260192a0e0828da 2024-11-15T14:15:53,034 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/B of e966eb2fe52084edc96ca33b9ecedaa2 into 06886368f70c4c1e9260192a0e0828da(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:53,034 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:53,034 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/B, priority=12, startTime=1731680152928; duration=0sec 2024-11-15T14:15:53,035 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:53,035 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:B 2024-11-15T14:15:53,037 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:53,037 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/567b316a5c7d4000993b5952f153a43f is 50, key is test_row_0/C:col10/1731680150870/Put/seqid=0 2024-11-15T14:15:53,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742139_1315 (size=12301) 2024-11-15T14:15:53,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742140_1316 (size=13255) 2024-11-15T14:15:53,093 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/567b316a5c7d4000993b5952f153a43f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/567b316a5c7d4000993b5952f153a43f 2024-11-15T14:15:53,100 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/C of e966eb2fe52084edc96ca33b9ecedaa2 into 567b316a5c7d4000993b5952f153a43f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:53,100 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:53,100 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/C, priority=12, startTime=1731680152928; duration=0sec 2024-11-15T14:15:53,100 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:53,100 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:C 2024-11-15T14:15:53,461 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/bf3d027153b74308837056eef9ca0b17 2024-11-15T14:15:53,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/2865d04dcda1459e8e0504b76bd87e06 is 50, key is test_row_0/B:col10/1731680151564/Put/seqid=0 2024-11-15T14:15:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T14:15:53,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742141_1317 (size=12301) 2024-11-15T14:15:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:53,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680213792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:53,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680213795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680213798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680213798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:53,903 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/2865d04dcda1459e8e0504b76bd87e06 2024-11-15T14:15:53,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:53,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680213912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680213912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:53,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680213912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:53,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/5b07836be0d94894aa03490ec12d93cf is 50, key is test_row_0/C:col10/1731680151564/Put/seqid=0 2024-11-15T14:15:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:53,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680213912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:53,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742142_1318 (size=12301) 2024-11-15T14:15:53,971 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/5b07836be0d94894aa03490ec12d93cf 2024-11-15T14:15:53,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/bf3d027153b74308837056eef9ca0b17 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/bf3d027153b74308837056eef9ca0b17 2024-11-15T14:15:53,994 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/bf3d027153b74308837056eef9ca0b17, entries=150, sequenceid=443, filesize=12.0 K 2024-11-15T14:15:53,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/2865d04dcda1459e8e0504b76bd87e06 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/2865d04dcda1459e8e0504b76bd87e06 2024-11-15T14:15:54,002 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/2865d04dcda1459e8e0504b76bd87e06, entries=150, sequenceid=443, filesize=12.0 K 2024-11-15T14:15:54,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/5b07836be0d94894aa03490ec12d93cf as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/5b07836be0d94894aa03490ec12d93cf 2024-11-15T14:15:54,010 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/5b07836be0d94894aa03490ec12d93cf, entries=150, sequenceid=443, filesize=12.0 K 2024-11-15T14:15:54,012 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e966eb2fe52084edc96ca33b9ecedaa2 in 1048ms, sequenceid=443, compaction requested=false 2024-11-15T14:15:54,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:54,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:54,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-15T14:15:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-15T14:15:54,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-15T14:15:54,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6770 sec 2024-11-15T14:15:54,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.6840 sec 2024-11-15T14:15:54,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T14:15:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:54,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731680153787, max timestamp=1731680154137Num uniques 859; 2024-11-15T14:15:54,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:15:54,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:54,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:54,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:54,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:54,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:54,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:15:54,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/ea45c9bb4c2b4242899f8e0041a127ab is 50, key is test_row_0/A:col10/1731680154137/Put/seqid=0 2024-11-15T14:15:54,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742143_1319 (size=12301) 2024-11-15T14:15:54,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680214225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680214227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680214230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680214231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680214342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680214352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680214352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680214352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T14:15:54,493 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:54,494 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-15T14:15:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T14:15:54,496 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:54,497 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:54,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:54,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680214564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/ea45c9bb4c2b4242899f8e0041a127ab 2024-11-15T14:15:54,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680214573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680214573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680214574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T14:15:54,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/245a04f14bfd4c46995ae53b02f2375b is 50, key is test_row_0/B:col10/1731680154137/Put/seqid=0 2024-11-15T14:15:54,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T14:15:54,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:54,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:54,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:54,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:54,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:54,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742144_1320 (size=12301) 2024-11-15T14:15:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T14:15:54,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T14:15:54,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:54,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:54,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:54,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:54,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:54,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680214881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680214893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680214893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680214893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:54,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T14:15:54,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:54,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:54,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:54,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:54,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:55,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/245a04f14bfd4c46995ae53b02f2375b 2024-11-15T14:15:55,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ca90baa87516499f8adb59f51dcdc9ff is 50, key is test_row_0/C:col10/1731680154137/Put/seqid=0 2024-11-15T14:15:55,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742145_1321 (size=12301) 2024-11-15T14:15:55,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ca90baa87516499f8adb59f51dcdc9ff 2024-11-15T14:15:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T14:15:55,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/ea45c9bb4c2b4242899f8e0041a127ab as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ea45c9bb4c2b4242899f8e0041a127ab 2024-11-15T14:15:55,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T14:15:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:55,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:55,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ea45c9bb4c2b4242899f8e0041a127ab, entries=150, sequenceid=467, filesize=12.0 K 2024-11-15T14:15:55,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/245a04f14bfd4c46995ae53b02f2375b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/245a04f14bfd4c46995ae53b02f2375b 2024-11-15T14:15:55,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/245a04f14bfd4c46995ae53b02f2375b, entries=150, sequenceid=467, filesize=12.0 K 2024-11-15T14:15:55,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ca90baa87516499f8adb59f51dcdc9ff as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ca90baa87516499f8adb59f51dcdc9ff 2024-11-15T14:15:55,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ca90baa87516499f8adb59f51dcdc9ff, entries=150, sequenceid=467, filesize=12.0 K 2024-11-15T14:15:55,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~91.69 KB/93890, heapSize ~267.14 KB/273552, currentSize=87.22 KB/89310 for e966eb2fe52084edc96ca33b9ecedaa2 in 1010ms, sequenceid=467, compaction requested=true 2024-11-15T14:15:55,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:55,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:55,148 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:55,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:55,148 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:55,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:55,150 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:55,150 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/B is initiating minor compaction (all files) 2024-11-15T14:15:55,150 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/B in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:55,150 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/06886368f70c4c1e9260192a0e0828da, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/2865d04dcda1459e8e0504b76bd87e06, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/245a04f14bfd4c46995ae53b02f2375b] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=37.0 K 2024-11-15T14:15:55,150 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:55,150 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/A is initiating minor compaction (all files) 2024-11-15T14:15:55,150 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/A in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:55,151 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f2ab4586d51c4755adb9ba952cad98c2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/bf3d027153b74308837056eef9ca0b17, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ea45c9bb4c2b4242899f8e0041a127ab] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=37.0 K 2024-11-15T14:15:55,152 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2ab4586d51c4755adb9ba952cad98c2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731680150870 2024-11-15T14:15:55,152 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 06886368f70c4c1e9260192a0e0828da, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731680150870 2024-11-15T14:15:55,153 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf3d027153b74308837056eef9ca0b17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731680151564 2024-11-15T14:15:55,154 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea45c9bb4c2b4242899f8e0041a127ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731680153787 2024-11-15T14:15:55,154 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2865d04dcda1459e8e0504b76bd87e06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731680151564 2024-11-15T14:15:55,154 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 245a04f14bfd4c46995ae53b02f2375b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731680153787 2024-11-15T14:15:55,173 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#A#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:55,173 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/b393225dfedf498a9bc6b6109a78e877 is 50, key is test_row_0/A:col10/1731680154137/Put/seqid=0 2024-11-15T14:15:55,188 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:55,188 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/aac23e04e85c467fa37718fa4dce867e is 50, key is test_row_0/B:col10/1731680154137/Put/seqid=0 2024-11-15T14:15:55,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742146_1322 (size=13357) 2024-11-15T14:15:55,238 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/b393225dfedf498a9bc6b6109a78e877 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b393225dfedf498a9bc6b6109a78e877 2024-11-15T14:15:55,250 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/A of e966eb2fe52084edc96ca33b9ecedaa2 into b393225dfedf498a9bc6b6109a78e877(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:55,252 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:55,252 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/A, priority=13, startTime=1731680155148; duration=0sec 2024-11-15T14:15:55,252 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:55,252 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:A 2024-11-15T14:15:55,252 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:55,256 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:55,256 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/C is initiating minor compaction (all files) 2024-11-15T14:15:55,256 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/C in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:55,256 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/567b316a5c7d4000993b5952f153a43f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/5b07836be0d94894aa03490ec12d93cf, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ca90baa87516499f8adb59f51dcdc9ff] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=37.0 K 2024-11-15T14:15:55,257 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 567b316a5c7d4000993b5952f153a43f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731680150870 2024-11-15T14:15:55,258 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b07836be0d94894aa03490ec12d93cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731680151564 2024-11-15T14:15:55,259 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca90baa87516499f8adb59f51dcdc9ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731680153787 2024-11-15T14:15:55,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742147_1323 (size=13357) 2024-11-15T14:15:55,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T14:15:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:55,281 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T14:15:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:55,290 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#C#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:55,291 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/21b2f76aa4514e72b177b679b493890a is 50, key is test_row_0/C:col10/1731680154137/Put/seqid=0 2024-11-15T14:15:55,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/053b4204d0284191b195bbfd3159ced4 is 50, key is test_row_0/A:col10/1731680154223/Put/seqid=0 2024-11-15T14:15:55,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742148_1324 (size=13357) 2024-11-15T14:15:55,368 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/21b2f76aa4514e72b177b679b493890a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/21b2f76aa4514e72b177b679b493890a 2024-11-15T14:15:55,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742149_1325 (size=12301) 2024-11-15T14:15:55,376 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/053b4204d0284191b195bbfd3159ced4 2024-11-15T14:15:55,378 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/C of e966eb2fe52084edc96ca33b9ecedaa2 into 21b2f76aa4514e72b177b679b493890a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:55,378 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:55,378 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/C, priority=13, startTime=1731680155149; duration=0sec 2024-11-15T14:15:55,378 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:55,379 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:C 2024-11-15T14:15:55,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/004ef3ab3925423cb80c0be415f32532 is 50, key is test_row_0/B:col10/1731680154223/Put/seqid=0 2024-11-15T14:15:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:55,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742150_1326 (size=12301) 2024-11-15T14:15:55,415 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/004ef3ab3925423cb80c0be415f32532 2024-11-15T14:15:55,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ad2d2be3aaee4adf8e593d0909a78cef is 50, key is test_row_0/C:col10/1731680154223/Put/seqid=0 2024-11-15T14:15:55,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680215432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680215437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680215441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742151_1327 (size=12301) 2024-11-15T14:15:55,459 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ad2d2be3aaee4adf8e593d0909a78cef 2024-11-15T14:15:55,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680215461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/053b4204d0284191b195bbfd3159ced4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/053b4204d0284191b195bbfd3159ced4 2024-11-15T14:15:55,484 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/053b4204d0284191b195bbfd3159ced4, entries=150, sequenceid=484, filesize=12.0 K 2024-11-15T14:15:55,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/004ef3ab3925423cb80c0be415f32532 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/004ef3ab3925423cb80c0be415f32532 2024-11-15T14:15:55,496 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/004ef3ab3925423cb80c0be415f32532, entries=150, sequenceid=484, filesize=12.0 K 2024-11-15T14:15:55,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ad2d2be3aaee4adf8e593d0909a78cef as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ad2d2be3aaee4adf8e593d0909a78cef 2024-11-15T14:15:55,501 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ad2d2be3aaee4adf8e593d0909a78cef, entries=150, sequenceid=484, filesize=12.0 K 2024-11-15T14:15:55,503 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for e966eb2fe52084edc96ca33b9ecedaa2 in 221ms, sequenceid=484, compaction requested=false 2024-11-15T14:15:55,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:55,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:55,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-15T14:15:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-15T14:15:55,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-15T14:15:55,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0070 sec 2024-11-15T14:15:55,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.0130 sec 2024-11-15T14:15:55,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T14:15:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:55,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:55,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:55,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:55,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:55,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:55,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:55,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/0f593afed44245269c19bab7c0826d27 is 50, key is test_row_0/A:col10/1731680155560/Put/seqid=0 2024-11-15T14:15:55,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742152_1328 (size=12301) 2024-11-15T14:15:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T14:15:55,632 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:15:55,632 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:15:55,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680215594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-15T14:15:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T14:15:55,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680215636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680215643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,647 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:15:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680215643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,649 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:15:55,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:15:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,672 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/aac23e04e85c467fa37718fa4dce867e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/aac23e04e85c467fa37718fa4dce867e 2024-11-15T14:15:55,679 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/B of e966eb2fe52084edc96ca33b9ecedaa2 into aac23e04e85c467fa37718fa4dce867e(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:55,679 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:55,679 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/B, priority=13, startTime=1731680155148; duration=0sec 2024-11-15T14:15:55,679 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:55,679 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:B 2024-11-15T14:15:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T14:15:55,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680215752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680215752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680215752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680215761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-15T14:15:55,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:55,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:55,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:55,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:55,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:55,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-15T14:15:55,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:55,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:55,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:55,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:55,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T14:15:55,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680215961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680215962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680215962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:55,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680215972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:56,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/0f593afed44245269c19bab7c0826d27 2024-11-15T14:15:56,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/358998a38b264b00af9fcddaf8411d57 is 50, key is test_row_0/B:col10/1731680155560/Put/seqid=0 2024-11-15T14:15:56,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742153_1329 (size=12301) 2024-11-15T14:15:56,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-15T14:15:56,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:56,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:56,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:56,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:56,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:56,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-15T14:15:56,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:56,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:56,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:56,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:56,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T14:15:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:56,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60976 deadline: 1731680216273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:56,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60932 deadline: 1731680216273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:56,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:56,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:56,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:56,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60982 deadline: 1731680216273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:56,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:15:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60950 deadline: 1731680216281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 2024-11-15T14:15:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:15:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:15:56,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-15T14:15:56,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:56,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:56,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:56,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:56,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:56,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/358998a38b264b00af9fcddaf8411d57 2024-11-15T14:15:56,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/7cb8fbaaa0c943179f33fb2100a97237 is 50, key is test_row_0/C:col10/1731680155560/Put/seqid=0 2024-11-15T14:15:56,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742154_1330 (size=12301) 2024-11-15T14:15:56,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/7cb8fbaaa0c943179f33fb2100a97237 2024-11-15T14:15:56,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/0f593afed44245269c19bab7c0826d27 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0f593afed44245269c19bab7c0826d27 2024-11-15T14:15:56,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0f593afed44245269c19bab7c0826d27, entries=150, sequenceid=504, filesize=12.0 K 2024-11-15T14:15:56,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-15T14:15:56,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:56,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:56,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:56,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:56,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:15:56,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/358998a38b264b00af9fcddaf8411d57 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/358998a38b264b00af9fcddaf8411d57 2024-11-15T14:15:56,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/358998a38b264b00af9fcddaf8411d57, entries=150, sequenceid=504, filesize=12.0 K 2024-11-15T14:15:56,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/7cb8fbaaa0c943179f33fb2100a97237 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/7cb8fbaaa0c943179f33fb2100a97237 2024-11-15T14:15:56,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/7cb8fbaaa0c943179f33fb2100a97237, entries=150, sequenceid=504, filesize=12.0 K 2024-11-15T14:15:56,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for e966eb2fe52084edc96ca33b9ecedaa2 in 1041ms, sequenceid=504, compaction requested=true 2024-11-15T14:15:56,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:56,603 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:56,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:15:56,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:56,604 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:56,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:15:56,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:56,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e966eb2fe52084edc96ca33b9ecedaa2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:15:56,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:56,607 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:56,607 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/A is initiating minor compaction (all files) 2024-11-15T14:15:56,607 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/A in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:56,607 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b393225dfedf498a9bc6b6109a78e877, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/053b4204d0284191b195bbfd3159ced4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0f593afed44245269c19bab7c0826d27] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=37.1 K 2024-11-15T14:15:56,609 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:56,609 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/B is initiating minor compaction (all files) 2024-11-15T14:15:56,610 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/B in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:56,610 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/aac23e04e85c467fa37718fa4dce867e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/004ef3ab3925423cb80c0be415f32532, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/358998a38b264b00af9fcddaf8411d57] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=37.1 K 2024-11-15T14:15:56,610 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b393225dfedf498a9bc6b6109a78e877, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731680153787 2024-11-15T14:15:56,613 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting aac23e04e85c467fa37718fa4dce867e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731680153787 2024-11-15T14:15:56,614 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 053b4204d0284191b195bbfd3159ced4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731680154223 2024-11-15T14:15:56,614 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 004ef3ab3925423cb80c0be415f32532, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731680154223 2024-11-15T14:15:56,616 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f593afed44245269c19bab7c0826d27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731680155436 2024-11-15T14:15:56,616 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 358998a38b264b00af9fcddaf8411d57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731680155436 2024-11-15T14:15:56,631 INFO [Thread-1087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1087. 2024-11-15T14:15:56,631 DEBUG [Thread-1087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:56,631 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,631 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:56,631 INFO [Thread-1085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1085. 2024-11-15T14:15:56,631 DEBUG [Thread-1085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:56,631 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,631 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:56,633 INFO [Thread-1079 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1079. 2024-11-15T14:15:56,633 DEBUG [Thread-1079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:56,633 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,633 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,633 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:56,634 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#A#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:56,635 INFO [Thread-1081 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1081. 2024-11-15T14:15:56,635 DEBUG [Thread-1081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:56,635 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,635 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,635 INFO [Thread-1083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1083. 2024-11-15T14:15:56,635 DEBUG [Thread-1083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:56,635 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,635 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/a23ffa54ae9a44a2b40098e2a7d602dc is 50, key is test_row_0/A:col10/1731680155560/Put/seqid=0 2024-11-15T14:15:56,635 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,635 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:56,635 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:56,638 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#B#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:56,638 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/480d6f94b72e4411a8928fd3dbf83fa2 is 50, key is test_row_0/B:col10/1731680155560/Put/seqid=0 2024-11-15T14:15:56,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742155_1331 (size=13459) 2024-11-15T14:15:56,649 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/a23ffa54ae9a44a2b40098e2a7d602dc as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/a23ffa54ae9a44a2b40098e2a7d602dc 2024-11-15T14:15:56,656 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/A of e966eb2fe52084edc96ca33b9ecedaa2 into a23ffa54ae9a44a2b40098e2a7d602dc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:56,656 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:56,656 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/A, priority=13, startTime=1731680156603; duration=0sec 2024-11-15T14:15:56,656 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:15:56,656 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:A 2024-11-15T14:15:56,656 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:15:56,660 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:15:56,661 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e966eb2fe52084edc96ca33b9ecedaa2/C is initiating minor compaction (all files) 2024-11-15T14:15:56,661 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e966eb2fe52084edc96ca33b9ecedaa2/C in TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:56,661 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/21b2f76aa4514e72b177b679b493890a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ad2d2be3aaee4adf8e593d0909a78cef, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/7cb8fbaaa0c943179f33fb2100a97237] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp, totalSize=37.1 K 2024-11-15T14:15:56,661 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21b2f76aa4514e72b177b679b493890a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731680153787 2024-11-15T14:15:56,662 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad2d2be3aaee4adf8e593d0909a78cef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731680154223 2024-11-15T14:15:56,665 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cb8fbaaa0c943179f33fb2100a97237, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1731680155436 2024-11-15T14:15:56,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742156_1332 (size=13459) 2024-11-15T14:15:56,675 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e966eb2fe52084edc96ca33b9ecedaa2#C#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:15:56,676 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/4f8b5b82b069409083164b4b10d5e1c4 is 50, key is test_row_0/C:col10/1731680155560/Put/seqid=0 2024-11-15T14:15:56,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742157_1333 (size=13459) 2024-11-15T14:15:56,710 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/4f8b5b82b069409083164b4b10d5e1c4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/4f8b5b82b069409083164b4b10d5e1c4 2024-11-15T14:15:56,716 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/C of e966eb2fe52084edc96ca33b9ecedaa2 into 4f8b5b82b069409083164b4b10d5e1c4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:56,716 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:56,717 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/C, priority=13, startTime=1731680156605; duration=0sec 2024-11-15T14:15:56,717 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:56,717 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:C 2024-11-15T14:15:56,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-15T14:15:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:56,734 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T14:15:56,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:15:56,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:56,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:15:56,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:56,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:15:56,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:15:56,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/7f524f698da44fb9a31cb7ec9dd26251 is 50, key is test_row_0/A:col10/1731680155606/Put/seqid=0 2024-11-15T14:15:56,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742158_1334 (size=12301) 2024-11-15T14:15:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T14:15:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:15:56,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. as already flushing 2024-11-15T14:15:56,785 INFO [Thread-1074 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1074. 2024-11-15T14:15:56,785 DEBUG [Thread-1074 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:56,786 DEBUG [Thread-1074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,786 DEBUG [Thread-1074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:56,787 INFO [Thread-1070 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1070. 2024-11-15T14:15:56,787 DEBUG [Thread-1070 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:56,787 DEBUG [Thread-1070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,788 DEBUG [Thread-1070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:56,795 INFO [Thread-1076 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1076. 2024-11-15T14:15:56,795 DEBUG [Thread-1076 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:56,795 DEBUG [Thread-1076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,795 DEBUG [Thread-1076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:56,798 INFO [Thread-1072 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1072. 2024-11-15T14:15:56,798 DEBUG [Thread-1072 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:15:56,798 DEBUG [Thread-1072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,799 DEBUG [Thread-1072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:15:56,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:15:57,078 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/480d6f94b72e4411a8928fd3dbf83fa2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/480d6f94b72e4411a8928fd3dbf83fa2 2024-11-15T14:15:57,086 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e966eb2fe52084edc96ca33b9ecedaa2/B of e966eb2fe52084edc96ca33b9ecedaa2 into 480d6f94b72e4411a8928fd3dbf83fa2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:15:57,086 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:57,086 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., storeName=e966eb2fe52084edc96ca33b9ecedaa2/B, priority=13, startTime=1731680156604; duration=0sec 2024-11-15T14:15:57,086 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:15:57,086 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e966eb2fe52084edc96ca33b9ecedaa2:B 2024-11-15T14:15:57,170 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/7f524f698da44fb9a31cb7ec9dd26251 2024-11-15T14:15:57,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/46f18c569a034827aa9f643b196179b1 is 50, key is test_row_0/B:col10/1731680155606/Put/seqid=0 2024-11-15T14:15:57,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742159_1335 (size=12301) 2024-11-15T14:15:57,581 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/46f18c569a034827aa9f643b196179b1 2024-11-15T14:15:57,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/5cdb0ee6ee6c4163a071ec9f18918eaa is 50, key is test_row_0/C:col10/1731680155606/Put/seqid=0 2024-11-15T14:15:57,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742160_1336 (size=12301) 2024-11-15T14:15:57,595 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/5cdb0ee6ee6c4163a071ec9f18918eaa 2024-11-15T14:15:57,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/7f524f698da44fb9a31cb7ec9dd26251 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/7f524f698da44fb9a31cb7ec9dd26251 2024-11-15T14:15:57,606 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/7f524f698da44fb9a31cb7ec9dd26251, entries=150, sequenceid=521, filesize=12.0 K 2024-11-15T14:15:57,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/46f18c569a034827aa9f643b196179b1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/46f18c569a034827aa9f643b196179b1 2024-11-15T14:15:57,613 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/46f18c569a034827aa9f643b196179b1, entries=150, sequenceid=521, filesize=12.0 K 2024-11-15T14:15:57,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/5cdb0ee6ee6c4163a071ec9f18918eaa as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/5cdb0ee6ee6c4163a071ec9f18918eaa 2024-11-15T14:15:57,619 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/5cdb0ee6ee6c4163a071ec9f18918eaa, entries=150, sequenceid=521, filesize=12.0 K 2024-11-15T14:15:57,620 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=26.84 KB/27480 for e966eb2fe52084edc96ca33b9ecedaa2 in 885ms, sequenceid=521, compaction requested=false 2024-11-15T14:15:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for e966eb2fe52084edc96ca33b9ecedaa2: 2024-11-15T14:15:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:15:57,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-15T14:15:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-15T14:15:57,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-15T14:15:57,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9740 sec 2024-11-15T14:15:57,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.9930 sec 2024-11-15T14:15:57,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T14:15:57,793 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:02,295 INFO [Thread-1068 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1068. 2024-11-15T14:16:02,295 DEBUG [Thread-1068 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:02,296 DEBUG [Thread-1068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:02,296 DEBUG [Thread-1068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:02,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:02,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T14:16:02,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 182 2024-11-15T14:16:02,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-15T14:16:02,297 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-15T14:16:02,297 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-15T14:16:02,297 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-15T14:16:02,297 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T14:16:02,297 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1966 2024-11-15T14:16:02,297 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1996 2024-11-15T14:16:02,297 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1932 2024-11-15T14:16:02,297 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1927 2024-11-15T14:16:02,297 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1990 2024-11-15T14:16:02,297 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T14:16:02,297 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T14:16:02,297 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T14:16:02,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:02,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:02,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:02,299 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T14:16:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T14:16:02,304 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680162303"}]},"ts":"1731680162303"} 2024-11-15T14:16:02,305 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T14:16:02,305 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T14:16:02,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T14:16:02,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e966eb2fe52084edc96ca33b9ecedaa2, UNASSIGN}] 2024-11-15T14:16:02,308 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e966eb2fe52084edc96ca33b9ecedaa2, UNASSIGN 2024-11-15T14:16:02,309 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=e966eb2fe52084edc96ca33b9ecedaa2, regionState=CLOSING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:16:02,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e966eb2fe52084edc96ca33b9ecedaa2, UNASSIGN because future has completed 2024-11-15T14:16:02,310 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T14:16:02,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:16:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T14:16:02,463 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:16:02,463 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T14:16:02,464 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing e966eb2fe52084edc96ca33b9ecedaa2, disabling compactions & flushes 2024-11-15T14:16:02,464 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:16:02,464 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:16:02,464 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. after waiting 0 ms 2024-11-15T14:16:02,464 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:16:02,464 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing e966eb2fe52084edc96ca33b9ecedaa2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T14:16:02,465 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=A 2024-11-15T14:16:02,465 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:02,465 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=B 2024-11-15T14:16:02,465 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:02,465 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e966eb2fe52084edc96ca33b9ecedaa2, store=C 2024-11-15T14:16:02,465 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:02,472 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/a630469443c44858a0894f55c515ff97 is 50, key is test_row_1/A:col10/1731680156793/Put/seqid=0 2024-11-15T14:16:02,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742161_1337 (size=9857) 2024-11-15T14:16:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T14:16:02,878 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/a630469443c44858a0894f55c515ff97 2024-11-15T14:16:02,885 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/379bca843f7e488a808b0f35672daca2 is 50, key is test_row_1/B:col10/1731680156793/Put/seqid=0 2024-11-15T14:16:02,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742162_1338 (size=9857) 2024-11-15T14:16:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T14:16:03,289 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/379bca843f7e488a808b0f35672daca2 2024-11-15T14:16:03,297 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ef557775a5464918b24aee6161c2799a is 50, key is test_row_1/C:col10/1731680156793/Put/seqid=0 2024-11-15T14:16:03,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742163_1339 (size=9857) 2024-11-15T14:16:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T14:16:03,704 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ef557775a5464918b24aee6161c2799a 2024-11-15T14:16:03,714 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/A/a630469443c44858a0894f55c515ff97 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/a630469443c44858a0894f55c515ff97 2024-11-15T14:16:03,718 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/a630469443c44858a0894f55c515ff97, entries=100, sequenceid=530, filesize=9.6 K 2024-11-15T14:16:03,719 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/B/379bca843f7e488a808b0f35672daca2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/379bca843f7e488a808b0f35672daca2 2024-11-15T14:16:03,724 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/379bca843f7e488a808b0f35672daca2, entries=100, sequenceid=530, filesize=9.6 K 2024-11-15T14:16:03,725 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/.tmp/C/ef557775a5464918b24aee6161c2799a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ef557775a5464918b24aee6161c2799a 2024-11-15T14:16:03,729 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ef557775a5464918b24aee6161c2799a, entries=100, sequenceid=530, filesize=9.6 K 2024-11-15T14:16:03,730 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e966eb2fe52084edc96ca33b9ecedaa2 in 1266ms, sequenceid=530, compaction requested=true 2024-11-15T14:16:03,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/35c244cc7d7b4c84a4719ed12ca98306, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/03c68fc8a6154b6cb3511553b4dd265a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f10a57d90fff496199af5d7b8802a552, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/02af1b821e344d078f76365bc1a73347, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/525f2bd0698940ca85318bf6f9e6415a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f60020252bf14dd382526e126530522d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/8cf038b420224a278aa030ed68713adf, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0c3868fc35e94cc68b9f1991b1e0bf2d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/d15d9cdfeee14cbe9f3860bb3d6c2cb1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/87bd88b592ee43f6b6745fe9f042be17, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/fd940a19a75246d08d0e063c20ea8ca8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/82233b87b21f458d9abd963bb606c408, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/55814f091ad8412d90dcb06aad0fc6e4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/45bf9b66da8e448c8e1e4d54ffff9a1e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ce224d9b10294a6cb295d550f2ec6bc8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b73d1747880e417f8783196644959492, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/216aefb8bdd94d6a8656df61d5ae99c4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/24924703ee0749d48e2c679a5ba5a572, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/9b2db47bb5db41c796bb8e9c43e18fc5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/9fd42392e8744fa38cb9c30b8e134a64, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0ffa71429fb24283916a56b7c71014ab, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/67ed6d66a0b44416b2a46d6000b49197, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ee66cc9a317f452cb18613c021b4a341, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/34b71eb89cc840989fafaf10109f51eb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b1bd4310c1464bdc8752fcedc3e4da0c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/439014429113463ba51ea5fbce13929b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/d9d819a2c910434a9015c75529992d0c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f2ab4586d51c4755adb9ba952cad98c2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/146023d470834305955575b50f567d51, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/bf3d027153b74308837056eef9ca0b17, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b393225dfedf498a9bc6b6109a78e877, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ea45c9bb4c2b4242899f8e0041a127ab, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/053b4204d0284191b195bbfd3159ced4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0f593afed44245269c19bab7c0826d27] to archive 2024-11-15T14:16:03,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:16:03,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/35c244cc7d7b4c84a4719ed12ca98306 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/35c244cc7d7b4c84a4719ed12ca98306 2024-11-15T14:16:03,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/03c68fc8a6154b6cb3511553b4dd265a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/03c68fc8a6154b6cb3511553b4dd265a 2024-11-15T14:16:03,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f10a57d90fff496199af5d7b8802a552 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f10a57d90fff496199af5d7b8802a552 2024-11-15T14:16:03,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/02af1b821e344d078f76365bc1a73347 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/02af1b821e344d078f76365bc1a73347 2024-11-15T14:16:03,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/525f2bd0698940ca85318bf6f9e6415a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/525f2bd0698940ca85318bf6f9e6415a 2024-11-15T14:16:03,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f60020252bf14dd382526e126530522d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f60020252bf14dd382526e126530522d 2024-11-15T14:16:03,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/8cf038b420224a278aa030ed68713adf to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/8cf038b420224a278aa030ed68713adf 2024-11-15T14:16:03,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0c3868fc35e94cc68b9f1991b1e0bf2d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0c3868fc35e94cc68b9f1991b1e0bf2d 2024-11-15T14:16:03,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/d15d9cdfeee14cbe9f3860bb3d6c2cb1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/d15d9cdfeee14cbe9f3860bb3d6c2cb1 2024-11-15T14:16:03,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/87bd88b592ee43f6b6745fe9f042be17 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/87bd88b592ee43f6b6745fe9f042be17 2024-11-15T14:16:03,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/fd940a19a75246d08d0e063c20ea8ca8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/fd940a19a75246d08d0e063c20ea8ca8 2024-11-15T14:16:03,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/82233b87b21f458d9abd963bb606c408 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/82233b87b21f458d9abd963bb606c408 2024-11-15T14:16:03,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/55814f091ad8412d90dcb06aad0fc6e4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/55814f091ad8412d90dcb06aad0fc6e4 2024-11-15T14:16:03,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/45bf9b66da8e448c8e1e4d54ffff9a1e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/45bf9b66da8e448c8e1e4d54ffff9a1e 2024-11-15T14:16:03,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ce224d9b10294a6cb295d550f2ec6bc8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ce224d9b10294a6cb295d550f2ec6bc8 2024-11-15T14:16:03,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b73d1747880e417f8783196644959492 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b73d1747880e417f8783196644959492 2024-11-15T14:16:03,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/216aefb8bdd94d6a8656df61d5ae99c4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/216aefb8bdd94d6a8656df61d5ae99c4 2024-11-15T14:16:03,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/24924703ee0749d48e2c679a5ba5a572 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/24924703ee0749d48e2c679a5ba5a572 2024-11-15T14:16:03,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/9b2db47bb5db41c796bb8e9c43e18fc5 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/9b2db47bb5db41c796bb8e9c43e18fc5 2024-11-15T14:16:03,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/9fd42392e8744fa38cb9c30b8e134a64 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/9fd42392e8744fa38cb9c30b8e134a64 2024-11-15T14:16:03,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0ffa71429fb24283916a56b7c71014ab to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0ffa71429fb24283916a56b7c71014ab 2024-11-15T14:16:03,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/67ed6d66a0b44416b2a46d6000b49197 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/67ed6d66a0b44416b2a46d6000b49197 2024-11-15T14:16:03,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ee66cc9a317f452cb18613c021b4a341 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ee66cc9a317f452cb18613c021b4a341 2024-11-15T14:16:03,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/34b71eb89cc840989fafaf10109f51eb to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/34b71eb89cc840989fafaf10109f51eb 2024-11-15T14:16:03,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b1bd4310c1464bdc8752fcedc3e4da0c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b1bd4310c1464bdc8752fcedc3e4da0c 2024-11-15T14:16:03,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/439014429113463ba51ea5fbce13929b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/439014429113463ba51ea5fbce13929b 2024-11-15T14:16:03,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/d9d819a2c910434a9015c75529992d0c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/d9d819a2c910434a9015c75529992d0c 2024-11-15T14:16:03,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f2ab4586d51c4755adb9ba952cad98c2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/f2ab4586d51c4755adb9ba952cad98c2 2024-11-15T14:16:03,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/146023d470834305955575b50f567d51 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/146023d470834305955575b50f567d51 2024-11-15T14:16:03,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/bf3d027153b74308837056eef9ca0b17 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/bf3d027153b74308837056eef9ca0b17 2024-11-15T14:16:03,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b393225dfedf498a9bc6b6109a78e877 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/b393225dfedf498a9bc6b6109a78e877 2024-11-15T14:16:03,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ea45c9bb4c2b4242899f8e0041a127ab to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/ea45c9bb4c2b4242899f8e0041a127ab 2024-11-15T14:16:03,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/053b4204d0284191b195bbfd3159ced4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/053b4204d0284191b195bbfd3159ced4 2024-11-15T14:16:03,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0f593afed44245269c19bab7c0826d27 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/0f593afed44245269c19bab7c0826d27 2024-11-15T14:16:03,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/7b6bad13e6b34ea7b00dfb9ae1422d7b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/064b6abadc06454d8a839654738c7398, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/dee74df3465d4329b7d2b6e4a42f6959, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/7fdcb3c4001744299aabb228496da9b3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/270c3d4506c049c8991686cd3842c4e9, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/84629ba08b8146e99f99b30a70db156f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/87613ffb399e4c659bd162f9661d8b34, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/83b6bfad6570489e9cd3168add9b33c2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e4eddf55544c46c8b1d6e5271def4f13, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/82482276eaba4199984eda2a77ab8b9f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/12f245dd7efe480eb3797cfac13fa320, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/469109e67dbe4d9faa43451d2d0ef5f6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e2dceb32ff3844419881eed39dba2968, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/0582cd1339f94a888259cecdf0803dce, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/ef1a15fc765c4f09a1ff65ea6b5c42e8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e908977a1b7e4b17a90830b5cbdf970e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/fc7888732d814f8eb9a6a4ee51903096, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/cea20b896490454d888eb959e51bb0bf, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/c029a002362547e6893359b173aa4f29, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/1e4bfb5925074df79c10037f14e91eaa, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/3ef2999b01fd47be84d81a509fef9e2a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/089b97e266984f2683bd6f86e6a6b7d4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/c5115c1c20c94077a7f0d57097e6d72e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/61afd9f2cf99461f90baee9fa2593344, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/9da440f8472b49c4b4380800fa2c4cc1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/4741b21f82da4ba08dc43ad4cbe60b69, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/2666b103d0764c5a90a96fc64adb5c6a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/06886368f70c4c1e9260192a0e0828da, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/d3bd1cda8ddf4390b35f2186a87e54c2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/2865d04dcda1459e8e0504b76bd87e06, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/aac23e04e85c467fa37718fa4dce867e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/245a04f14bfd4c46995ae53b02f2375b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/004ef3ab3925423cb80c0be415f32532, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/358998a38b264b00af9fcddaf8411d57] to archive 2024-11-15T14:16:03,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:16:03,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/7b6bad13e6b34ea7b00dfb9ae1422d7b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/7b6bad13e6b34ea7b00dfb9ae1422d7b 2024-11-15T14:16:03,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/064b6abadc06454d8a839654738c7398 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/064b6abadc06454d8a839654738c7398 2024-11-15T14:16:03,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/dee74df3465d4329b7d2b6e4a42f6959 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/dee74df3465d4329b7d2b6e4a42f6959 2024-11-15T14:16:03,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/7fdcb3c4001744299aabb228496da9b3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/7fdcb3c4001744299aabb228496da9b3 2024-11-15T14:16:03,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/270c3d4506c049c8991686cd3842c4e9 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/270c3d4506c049c8991686cd3842c4e9 2024-11-15T14:16:03,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/84629ba08b8146e99f99b30a70db156f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/84629ba08b8146e99f99b30a70db156f 2024-11-15T14:16:03,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/87613ffb399e4c659bd162f9661d8b34 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/87613ffb399e4c659bd162f9661d8b34 2024-11-15T14:16:03,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/83b6bfad6570489e9cd3168add9b33c2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/83b6bfad6570489e9cd3168add9b33c2 2024-11-15T14:16:03,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e4eddf55544c46c8b1d6e5271def4f13 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e4eddf55544c46c8b1d6e5271def4f13 2024-11-15T14:16:03,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/82482276eaba4199984eda2a77ab8b9f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/82482276eaba4199984eda2a77ab8b9f 2024-11-15T14:16:03,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/12f245dd7efe480eb3797cfac13fa320 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/12f245dd7efe480eb3797cfac13fa320 2024-11-15T14:16:03,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/469109e67dbe4d9faa43451d2d0ef5f6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/469109e67dbe4d9faa43451d2d0ef5f6 2024-11-15T14:16:03,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e2dceb32ff3844419881eed39dba2968 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e2dceb32ff3844419881eed39dba2968 2024-11-15T14:16:03,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/0582cd1339f94a888259cecdf0803dce to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/0582cd1339f94a888259cecdf0803dce 2024-11-15T14:16:03,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/ef1a15fc765c4f09a1ff65ea6b5c42e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/ef1a15fc765c4f09a1ff65ea6b5c42e8 2024-11-15T14:16:03,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e908977a1b7e4b17a90830b5cbdf970e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/e908977a1b7e4b17a90830b5cbdf970e 2024-11-15T14:16:03,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/fc7888732d814f8eb9a6a4ee51903096 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/fc7888732d814f8eb9a6a4ee51903096 2024-11-15T14:16:03,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/cea20b896490454d888eb959e51bb0bf to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/cea20b896490454d888eb959e51bb0bf 2024-11-15T14:16:03,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/c029a002362547e6893359b173aa4f29 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/c029a002362547e6893359b173aa4f29 2024-11-15T14:16:03,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/1e4bfb5925074df79c10037f14e91eaa to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/1e4bfb5925074df79c10037f14e91eaa 2024-11-15T14:16:03,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/3ef2999b01fd47be84d81a509fef9e2a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/3ef2999b01fd47be84d81a509fef9e2a 2024-11-15T14:16:03,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/089b97e266984f2683bd6f86e6a6b7d4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/089b97e266984f2683bd6f86e6a6b7d4 2024-11-15T14:16:03,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/c5115c1c20c94077a7f0d57097e6d72e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/c5115c1c20c94077a7f0d57097e6d72e 2024-11-15T14:16:03,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/61afd9f2cf99461f90baee9fa2593344 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/61afd9f2cf99461f90baee9fa2593344 2024-11-15T14:16:03,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/9da440f8472b49c4b4380800fa2c4cc1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/9da440f8472b49c4b4380800fa2c4cc1 2024-11-15T14:16:03,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/4741b21f82da4ba08dc43ad4cbe60b69 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/4741b21f82da4ba08dc43ad4cbe60b69 2024-11-15T14:16:03,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/2666b103d0764c5a90a96fc64adb5c6a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/2666b103d0764c5a90a96fc64adb5c6a 2024-11-15T14:16:03,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/06886368f70c4c1e9260192a0e0828da to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/06886368f70c4c1e9260192a0e0828da 2024-11-15T14:16:03,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/d3bd1cda8ddf4390b35f2186a87e54c2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/d3bd1cda8ddf4390b35f2186a87e54c2 2024-11-15T14:16:03,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/2865d04dcda1459e8e0504b76bd87e06 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/2865d04dcda1459e8e0504b76bd87e06 2024-11-15T14:16:03,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/aac23e04e85c467fa37718fa4dce867e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/aac23e04e85c467fa37718fa4dce867e 2024-11-15T14:16:03,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/245a04f14bfd4c46995ae53b02f2375b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/245a04f14bfd4c46995ae53b02f2375b 2024-11-15T14:16:03,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/004ef3ab3925423cb80c0be415f32532 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/004ef3ab3925423cb80c0be415f32532 2024-11-15T14:16:03,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/358998a38b264b00af9fcddaf8411d57 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/358998a38b264b00af9fcddaf8411d57 2024-11-15T14:16:03,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/308a8ab5ab494bf88dc4bb442d0c2f42, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/f216676024b7476eacf2c254f9aa0ae2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/50ffdcf29d0549069702e1b2d4872bd2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/f4f4e4a4a5344b799309aef8e31d9b5b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ad6d61c70719499585bd0d9a164349a1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/6df4b46d799b42e0888cd615d8dc02a1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/454c2ad055b44b2cb0254423d1524ce5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/2d194672ec0847c68e13b70184e51293, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/01e55b1df0154ba48d1c08101b243b5e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/0f5d1fa3551a470e911037b1b5513466, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/9e6441ccbdd749f2af0b6cd41fbf8b6c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/1ec7e00015ef493f9a43ba58e8312c9f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/2d5a69202a044c0d88e467a9eb5b6f4d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/025c454535ef49818a25d6aecef710d8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/59b40855d8f840a9be67fa84131f6421, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/3abc5f89cfe54347b152c9aaf6bf741b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ec10bc6981844030b3356a236c751656, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/3e4ba79234bb4f0fa4b36d462eab425b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/e20791e39ae24de5b3a003ea63ca54c2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/fda15d27d9a94ac6a93ab12eae6d9c5b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ebcdac0058324f08927938f176806aaa, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/8cbe7c236e62435d9cdefb42f74e0783, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/97e9b1604e3e4bc19bf3395073384072, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/a89d500d910d402b82469936ce7e2718, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/fdc1200dc88b409287beac962d3d1dd4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/0eb2ac2902034a24b73a9a75a3c6384f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/00868a69fd644e16bf57c7a579032a66, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/567b316a5c7d4000993b5952f153a43f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/7911b2ee86334be0a482a5aa7651122a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/5b07836be0d94894aa03490ec12d93cf, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/21b2f76aa4514e72b177b679b493890a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ca90baa87516499f8adb59f51dcdc9ff, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ad2d2be3aaee4adf8e593d0909a78cef, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/7cb8fbaaa0c943179f33fb2100a97237] to archive 2024-11-15T14:16:03,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:16:03,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/308a8ab5ab494bf88dc4bb442d0c2f42 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/308a8ab5ab494bf88dc4bb442d0c2f42 2024-11-15T14:16:03,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/f216676024b7476eacf2c254f9aa0ae2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/f216676024b7476eacf2c254f9aa0ae2 2024-11-15T14:16:03,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/50ffdcf29d0549069702e1b2d4872bd2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/50ffdcf29d0549069702e1b2d4872bd2 2024-11-15T14:16:03,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/f4f4e4a4a5344b799309aef8e31d9b5b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/f4f4e4a4a5344b799309aef8e31d9b5b 2024-11-15T14:16:03,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ad6d61c70719499585bd0d9a164349a1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ad6d61c70719499585bd0d9a164349a1 2024-11-15T14:16:03,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/6df4b46d799b42e0888cd615d8dc02a1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/6df4b46d799b42e0888cd615d8dc02a1 2024-11-15T14:16:03,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/454c2ad055b44b2cb0254423d1524ce5 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/454c2ad055b44b2cb0254423d1524ce5 2024-11-15T14:16:03,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/2d194672ec0847c68e13b70184e51293 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/2d194672ec0847c68e13b70184e51293 2024-11-15T14:16:03,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/01e55b1df0154ba48d1c08101b243b5e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/01e55b1df0154ba48d1c08101b243b5e 2024-11-15T14:16:03,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/0f5d1fa3551a470e911037b1b5513466 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/0f5d1fa3551a470e911037b1b5513466 2024-11-15T14:16:03,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/9e6441ccbdd749f2af0b6cd41fbf8b6c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/9e6441ccbdd749f2af0b6cd41fbf8b6c 2024-11-15T14:16:03,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/1ec7e00015ef493f9a43ba58e8312c9f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/1ec7e00015ef493f9a43ba58e8312c9f 2024-11-15T14:16:03,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/2d5a69202a044c0d88e467a9eb5b6f4d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/2d5a69202a044c0d88e467a9eb5b6f4d 2024-11-15T14:16:03,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/025c454535ef49818a25d6aecef710d8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/025c454535ef49818a25d6aecef710d8 2024-11-15T14:16:03,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/59b40855d8f840a9be67fa84131f6421 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/59b40855d8f840a9be67fa84131f6421 2024-11-15T14:16:03,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/3abc5f89cfe54347b152c9aaf6bf741b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/3abc5f89cfe54347b152c9aaf6bf741b 2024-11-15T14:16:03,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ec10bc6981844030b3356a236c751656 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ec10bc6981844030b3356a236c751656 2024-11-15T14:16:03,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/3e4ba79234bb4f0fa4b36d462eab425b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/3e4ba79234bb4f0fa4b36d462eab425b 2024-11-15T14:16:03,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/e20791e39ae24de5b3a003ea63ca54c2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/e20791e39ae24de5b3a003ea63ca54c2 2024-11-15T14:16:03,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/fda15d27d9a94ac6a93ab12eae6d9c5b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/fda15d27d9a94ac6a93ab12eae6d9c5b 2024-11-15T14:16:03,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ebcdac0058324f08927938f176806aaa to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ebcdac0058324f08927938f176806aaa 2024-11-15T14:16:03,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/8cbe7c236e62435d9cdefb42f74e0783 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/8cbe7c236e62435d9cdefb42f74e0783 2024-11-15T14:16:03,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/97e9b1604e3e4bc19bf3395073384072 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/97e9b1604e3e4bc19bf3395073384072 2024-11-15T14:16:03,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/a89d500d910d402b82469936ce7e2718 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/a89d500d910d402b82469936ce7e2718 2024-11-15T14:16:03,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/fdc1200dc88b409287beac962d3d1dd4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/fdc1200dc88b409287beac962d3d1dd4 2024-11-15T14:16:03,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/0eb2ac2902034a24b73a9a75a3c6384f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/0eb2ac2902034a24b73a9a75a3c6384f 2024-11-15T14:16:03,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/00868a69fd644e16bf57c7a579032a66 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/00868a69fd644e16bf57c7a579032a66 2024-11-15T14:16:03,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/567b316a5c7d4000993b5952f153a43f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/567b316a5c7d4000993b5952f153a43f 2024-11-15T14:16:03,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/7911b2ee86334be0a482a5aa7651122a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/7911b2ee86334be0a482a5aa7651122a 2024-11-15T14:16:03,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/5b07836be0d94894aa03490ec12d93cf to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/5b07836be0d94894aa03490ec12d93cf 2024-11-15T14:16:03,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/21b2f76aa4514e72b177b679b493890a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/21b2f76aa4514e72b177b679b493890a 2024-11-15T14:16:03,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ca90baa87516499f8adb59f51dcdc9ff to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ca90baa87516499f8adb59f51dcdc9ff 2024-11-15T14:16:03,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ad2d2be3aaee4adf8e593d0909a78cef to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ad2d2be3aaee4adf8e593d0909a78cef 2024-11-15T14:16:03,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/7cb8fbaaa0c943179f33fb2100a97237 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/7cb8fbaaa0c943179f33fb2100a97237 2024-11-15T14:16:03,862 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/recovered.edits/533.seqid, newMaxSeqId=533, maxSeqId=1 2024-11-15T14:16:03,863 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. 2024-11-15T14:16:03,863 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for e966eb2fe52084edc96ca33b9ecedaa2: Waiting for close lock at 1731680162464Running coprocessor pre-close hooks at 1731680162464Disabling compacts and flushes for region at 1731680162464Disabling writes for close at 1731680162464Obtaining lock to block concurrent updates at 1731680162464Preparing flush snapshotting stores in e966eb2fe52084edc96ca33b9ecedaa2 at 1731680162464Finished memstore snapshotting TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731680162465 (+1 ms)Flushing stores of TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2. at 1731680162466 (+1 ms)Flushing e966eb2fe52084edc96ca33b9ecedaa2/A: creating writer at 1731680162466Flushing e966eb2fe52084edc96ca33b9ecedaa2/A: appending metadata at 1731680162471 (+5 ms)Flushing e966eb2fe52084edc96ca33b9ecedaa2/A: closing flushed file at 1731680162471Flushing e966eb2fe52084edc96ca33b9ecedaa2/B: creating writer at 1731680162882 (+411 ms)Flushing e966eb2fe52084edc96ca33b9ecedaa2/B: appending metadata at 1731680162884 (+2 ms)Flushing e966eb2fe52084edc96ca33b9ecedaa2/B: closing flushed file at 1731680162884Flushing e966eb2fe52084edc96ca33b9ecedaa2/C: creating writer at 1731680163294 (+410 ms)Flushing e966eb2fe52084edc96ca33b9ecedaa2/C: appending metadata at 1731680163297 (+3 ms)Flushing e966eb2fe52084edc96ca33b9ecedaa2/C: closing flushed file at 1731680163297Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@140fd96a: reopening flushed file at 1731680163713 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bfcfc1a: reopening flushed file at 1731680163718 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60330da6: reopening flushed file at 1731680163724 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e966eb2fe52084edc96ca33b9ecedaa2 in 1266ms, sequenceid=530, compaction requested=true at 1731680163730 (+6 ms)Writing region close event to WAL at 1731680163860 (+130 ms)Running coprocessor post-close hooks at 1731680163863 (+3 ms)Closed at 1731680163863 2024-11-15T14:16:03,865 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:16:03,866 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=e966eb2fe52084edc96ca33b9ecedaa2, regionState=CLOSED 2024-11-15T14:16:03,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:16:03,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-15T14:16:03,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure e966eb2fe52084edc96ca33b9ecedaa2, server=26754e923c4a,40177,1731680077530 in 1.5580 sec 2024-11-15T14:16:03,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-15T14:16:03,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e966eb2fe52084edc96ca33b9ecedaa2, UNASSIGN in 1.5630 sec 2024-11-15T14:16:03,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-15T14:16:03,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5660 sec 2024-11-15T14:16:03,875 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680163875"}]},"ts":"1731680163875"} 2024-11-15T14:16:03,876 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T14:16:03,876 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T14:16:03,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5780 sec 2024-11-15T14:16:04,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T14:16:04,462 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:04,463 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T14:16:04,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:04,464 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:04,465 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-15T14:16:04,468 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:16:04,470 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/recovered.edits] 2024-11-15T14:16:04,473 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/7f524f698da44fb9a31cb7ec9dd26251 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/7f524f698da44fb9a31cb7ec9dd26251 2024-11-15T14:16:04,475 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/a23ffa54ae9a44a2b40098e2a7d602dc to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/a23ffa54ae9a44a2b40098e2a7d602dc 2024-11-15T14:16:04,476 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/a630469443c44858a0894f55c515ff97 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/A/a630469443c44858a0894f55c515ff97 2024-11-15T14:16:04,480 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/379bca843f7e488a808b0f35672daca2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/379bca843f7e488a808b0f35672daca2 2024-11-15T14:16:04,481 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/46f18c569a034827aa9f643b196179b1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/46f18c569a034827aa9f643b196179b1 2024-11-15T14:16:04,483 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/480d6f94b72e4411a8928fd3dbf83fa2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/B/480d6f94b72e4411a8928fd3dbf83fa2 2024-11-15T14:16:04,487 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/4f8b5b82b069409083164b4b10d5e1c4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/4f8b5b82b069409083164b4b10d5e1c4 2024-11-15T14:16:04,489 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/5cdb0ee6ee6c4163a071ec9f18918eaa to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/5cdb0ee6ee6c4163a071ec9f18918eaa 2024-11-15T14:16:04,491 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ef557775a5464918b24aee6161c2799a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/C/ef557775a5464918b24aee6161c2799a 2024-11-15T14:16:04,494 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/recovered.edits/533.seqid to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2/recovered.edits/533.seqid 2024-11-15T14:16:04,495 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e966eb2fe52084edc96ca33b9ecedaa2 2024-11-15T14:16:04,495 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T14:16:04,498 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:04,503 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T14:16:04,506 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T14:16:04,507 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:04,507 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T14:16:04,507 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731680164507"}]},"ts":"9223372036854775807"} 2024-11-15T14:16:04,510 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T14:16:04,510 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e966eb2fe52084edc96ca33b9ecedaa2, NAME => 'TestAcidGuarantees,,1731680134411.e966eb2fe52084edc96ca33b9ecedaa2.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T14:16:04,510 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T14:16:04,510 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731680164510"}]},"ts":"9223372036854775807"} 2024-11-15T14:16:04,512 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T14:16:04,513 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:04,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-11-15T14:16:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-15T14:16:04,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T14:16:04,575 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:04,582 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 254), OpenFileDescriptor=453 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 696) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=260 (was 263) 2024-11-15T14:16:04,591 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=260 2024-11-15T14:16:04,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T14:16:04,592 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T14:16:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:04,594 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T14:16:04,594 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:04,594 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-15T14:16:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T14:16:04,595 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T14:16:04,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742164_1340 (size=1061) 2024-11-15T14:16:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T14:16:04,811 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T14:16:04,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T14:16:05,005 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931 2024-11-15T14:16:05,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742165_1341 (size=53) 2024-11-15T14:16:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T14:16:05,415 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:16:05,415 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a6c59e08ce89fd58e535f1c83edb91e8, disabling compactions & flushes 2024-11-15T14:16:05,415 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:05,415 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:05,415 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. after waiting 0 ms 2024-11-15T14:16:05,415 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:05,415 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:05,415 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a6c59e08ce89fd58e535f1c83edb91e8: Waiting for close lock at 1731680165415Disabling compacts and flushes for region at 1731680165415Disabling writes for close at 1731680165415Writing region close event to WAL at 1731680165415Closed at 1731680165415 2024-11-15T14:16:05,416 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T14:16:05,417 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731680165416"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731680165416"}]},"ts":"1731680165416"} 2024-11-15T14:16:05,419 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T14:16:05,420 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T14:16:05,420 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680165420"}]},"ts":"1731680165420"} 2024-11-15T14:16:05,422 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T14:16:05,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, ASSIGN}] 2024-11-15T14:16:05,424 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, ASSIGN 2024-11-15T14:16:05,425 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, ASSIGN; state=OFFLINE, location=26754e923c4a,40177,1731680077530; forceNewPlan=false, retain=false 2024-11-15T14:16:05,576 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a6c59e08ce89fd58e535f1c83edb91e8, regionState=OPENING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:16:05,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, ASSIGN because future has completed 2024-11-15T14:16:05,581 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:16:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T14:16:05,738 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:05,739 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} 2024-11-15T14:16:05,739 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:05,739 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:16:05,739 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:05,739 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:05,741 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:05,742 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:16:05,743 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6c59e08ce89fd58e535f1c83edb91e8 columnFamilyName A 2024-11-15T14:16:05,743 DEBUG [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:05,743 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.HStore(327): Store=a6c59e08ce89fd58e535f1c83edb91e8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:16:05,744 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:05,745 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:16:05,745 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6c59e08ce89fd58e535f1c83edb91e8 columnFamilyName B 2024-11-15T14:16:05,745 DEBUG [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:05,746 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.HStore(327): Store=a6c59e08ce89fd58e535f1c83edb91e8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:16:05,746 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:05,747 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:16:05,747 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6c59e08ce89fd58e535f1c83edb91e8 columnFamilyName C 2024-11-15T14:16:05,747 DEBUG [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:05,748 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.HStore(327): Store=a6c59e08ce89fd58e535f1c83edb91e8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:16:05,748 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:05,748 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:05,749 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:05,749 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:05,750 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:05,750 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:05,750 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T14:16:05,752 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:05,754 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T14:16:05,754 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened a6c59e08ce89fd58e535f1c83edb91e8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67354459, jitterRate=0.003659650683403015}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T14:16:05,754 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:05,755 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for a6c59e08ce89fd58e535f1c83edb91e8: Running coprocessor pre-open hook at 1731680165739Writing region info on filesystem at 1731680165739Initializing all the Stores at 1731680165740 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680165740Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680165741 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680165741Cleaning up temporary data from old regions at 1731680165750 (+9 ms)Running coprocessor post-open hooks at 1731680165754 (+4 ms)Region opened successfully at 1731680165755 (+1 ms) 2024-11-15T14:16:05,756 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., pid=95, masterSystemTime=1731680165735 2024-11-15T14:16:05,758 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:05,758 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:05,759 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a6c59e08ce89fd58e535f1c83edb91e8, regionState=OPEN, openSeqNum=2, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:16:05,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:16:05,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-15T14:16:05,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 in 180 msec 2024-11-15T14:16:05,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-15T14:16:05,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, ASSIGN in 341 msec 2024-11-15T14:16:05,767 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T14:16:05,767 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680165767"}]},"ts":"1731680165767"} 2024-11-15T14:16:05,769 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T14:16:05,769 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T14:16:05,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1770 sec 2024-11-15T14:16:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T14:16:06,742 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:06,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:06,743 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:06,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:06,747 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a9c651d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:06,749 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508cc606, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:06,750 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:06,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:06,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:06,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:06,753 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T14:16:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 26754e923c4a,35807,1731680075796 2024-11-15T14:16:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@39fe68ac 2024-11-15T14:16:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T14:16:06,757 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T14:16:06,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T14:16:06,760 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T14:16:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:06,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742166_1342 (size=1097) 2024-11-15T14:16:07,182 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T14:16:07,182 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T14:16:07,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T14:16:07,189 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, REOPEN/MOVE}] 2024-11-15T14:16:07,190 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, REOPEN/MOVE 2024-11-15T14:16:07,191 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=a6c59e08ce89fd58e535f1c83edb91e8, regionState=CLOSING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:16:07,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, REOPEN/MOVE because future has completed 2024-11-15T14:16:07,197 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T14:16:07,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:16:07,349 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,349 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T14:16:07,349 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing a6c59e08ce89fd58e535f1c83edb91e8, disabling compactions & flushes 2024-11-15T14:16:07,350 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:07,350 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:07,350 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. after waiting 0 ms 2024-11-15T14:16:07,350 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:07,354 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T14:16:07,355 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:07,355 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for a6c59e08ce89fd58e535f1c83edb91e8: Waiting for close lock at 1731680167349Running coprocessor pre-close hooks at 1731680167349Disabling compacts and flushes for region at 1731680167349Disabling writes for close at 1731680167350 (+1 ms)Writing region close event to WAL at 1731680167350Running coprocessor post-close hooks at 1731680167355 (+5 ms)Closed at 1731680167355 2024-11-15T14:16:07,355 WARN [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: a6c59e08ce89fd58e535f1c83edb91e8 to self. 2024-11-15T14:16:07,358 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,358 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=a6c59e08ce89fd58e535f1c83edb91e8, regionState=CLOSED 2024-11-15T14:16:07,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:16:07,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-15T14:16:07,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 in 164 msec 2024-11-15T14:16:07,364 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, REOPEN/MOVE; state=CLOSED, location=26754e923c4a,40177,1731680077530; forceNewPlan=false, retain=true 2024-11-15T14:16:07,515 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=a6c59e08ce89fd58e535f1c83edb91e8, regionState=OPENING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:16:07,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, REOPEN/MOVE because future has completed 2024-11-15T14:16:07,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:16:07,674 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:07,674 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} 2024-11-15T14:16:07,674 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,674 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:16:07,674 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,674 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,675 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,676 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:16:07,677 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6c59e08ce89fd58e535f1c83edb91e8 columnFamilyName A 2024-11-15T14:16:07,678 DEBUG [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:07,678 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.HStore(327): Store=a6c59e08ce89fd58e535f1c83edb91e8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:16:07,679 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,679 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:16:07,679 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6c59e08ce89fd58e535f1c83edb91e8 columnFamilyName B 2024-11-15T14:16:07,679 DEBUG [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:07,680 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.HStore(327): Store=a6c59e08ce89fd58e535f1c83edb91e8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:16:07,680 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,680 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:16:07,680 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6c59e08ce89fd58e535f1c83edb91e8 columnFamilyName C 2024-11-15T14:16:07,680 DEBUG [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:07,681 INFO [StoreOpener-a6c59e08ce89fd58e535f1c83edb91e8-1 {}] regionserver.HStore(327): Store=a6c59e08ce89fd58e535f1c83edb91e8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:16:07,681 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:07,681 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,682 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,682 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,684 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,684 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,685 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T14:16:07,686 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,687 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened a6c59e08ce89fd58e535f1c83edb91e8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71155966, jitterRate=0.06030651926994324}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T14:16:07,687 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,688 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for a6c59e08ce89fd58e535f1c83edb91e8: Running coprocessor pre-open hook at 1731680167674Writing region info on filesystem at 1731680167674Initializing all the Stores at 1731680167675 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680167675Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680167675Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680167675Cleaning up temporary data from old regions at 1731680167684 (+9 ms)Running coprocessor post-open hooks at 1731680167687 (+3 ms)Region opened successfully at 1731680167687 2024-11-15T14:16:07,688 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., pid=100, masterSystemTime=1731680167671 2024-11-15T14:16:07,690 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:07,690 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:07,690 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=a6c59e08ce89fd58e535f1c83edb91e8, regionState=OPEN, openSeqNum=5, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:16:07,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:16:07,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-15T14:16:07,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 in 176 msec 2024-11-15T14:16:07,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-15T14:16:07,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, REOPEN/MOVE in 505 msec 2024-11-15T14:16:07,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-15T14:16:07,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-11-15T14:16:07,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 938 msec 2024-11-15T14:16:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-15T14:16:07,701 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:07,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:07,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:07,702 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64223b25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:07,704 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@abc4eeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:07,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:07,705 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70113e71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:07,706 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74fa3850, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:07,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:07,708 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b756229, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:07,709 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a9f2b97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:07,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:07,711 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48e77add, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:07,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:07,712 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2caacafd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:07,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:07,714 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca8c76b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:07,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:07,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:07,715 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27efe49e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:07,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:07,717 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:07,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:07,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:07,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a643a48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:07,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:07,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:07,718 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40fb1ba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,719 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:07,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:07,719 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:07,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e58e80e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:07,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:07,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:07,721 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c6301c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:07,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:07,726 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1883b802, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:07,728 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40d8fb3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:07,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:07,732 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2465fe11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:07,734 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4732c587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,739 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:07,739 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:07,740 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8f013d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:07,741 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15e1b8ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:07,742 DEBUG [Thread-1552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:07,743 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:07,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:07,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:07,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:07,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:16:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:16:07,751 DEBUG [Thread-1571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:07,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:16:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:07,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T14:16:07,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:07,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:07,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:07,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:07,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:07,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:07,759 DEBUG [Thread-1565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:07,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:16:07,767 DEBUG [Thread-1569 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:07,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:16:07,803 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:07,803 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:07,804 DEBUG [Thread-1560 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:07,805 DEBUG [Thread-1560 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:07,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-15T14:16:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:16:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T14:16:07,809 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:07,809 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:07,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:07,815 DEBUG [Thread-1563 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:07,816 DEBUG [Thread-1563 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:07,819 DEBUG [Thread-1563 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:07,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:07,822 DEBUG [Thread-1563 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:16:07,824 DEBUG [Thread-1567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:07,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:16:07,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155c3aa1b29c844212aae8a8389aea4fad_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680167756/Put/seqid=0 2024-11-15T14:16:07,843 DEBUG [Thread-1554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:07,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:16:07,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680167806, max timestamp=1731680167847Num uniques 859; 2024-11-15T14:16:07,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680167806, max timestamp=1731680167847Num uniques 859; 2024-11-15T14:16:07,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:16:07,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:16:07,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680167806, max timestamp=1731680167847Num uniques 859; 2024-11-15T14:16:07,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:16:07,863 DEBUG [Thread-1558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:07,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:16:07,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742167_1343 (size=14594) 2024-11-15T14:16:07,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:07,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680227888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:07,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:07,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:07,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T14:16:07,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680227898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:07,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:07,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680227898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:07,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680227901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:07,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:07,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680227903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:07,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:07,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:07,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:07,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:07,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:07,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:07,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:07,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:07,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:07,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:07,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:07,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:07,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680228012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680228021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680228021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680228022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680228022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:08,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:08,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:08,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:08,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T14:16:08,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680228233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:08,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:08,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:08,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:08,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680228252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680228252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680228252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680228253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,292 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:08,297 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155c3aa1b29c844212aae8a8389aea4fad_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155c3aa1b29c844212aae8a8389aea4fad_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:08,301 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/03c064cd6d594b9aac1e0f8370cf6e28, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:08,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/03c064cd6d594b9aac1e0f8370cf6e28 is 175, key is test_row_0/A:col10/1731680167756/Put/seqid=0 2024-11-15T14:16:08,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742168_1344 (size=39545) 2024-11-15T14:16:08,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:08,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:08,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:08,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:08,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T14:16:08,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:08,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:08,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:08,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:08,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680228556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680228583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680228592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680228592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680228592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:08,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:08,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:08,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:08,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:08,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,781 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/03c064cd6d594b9aac1e0f8370cf6e28 2024-11-15T14:16:08,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/0bf6989de027496485ef5f1b1101ee13 is 50, key is test_row_0/B:col10/1731680167756/Put/seqid=0 2024-11-15T14:16:08,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742169_1345 (size=9657) 2024-11-15T14:16:08,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:08,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:08,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:08,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:08,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T14:16:09,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:09,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:09,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:09,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:09,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680229094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:09,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680229114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:09,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680229121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:09,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680229122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:09,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680229122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:09,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:09,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:09,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:09,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:09,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/0bf6989de027496485ef5f1b1101ee13 2024-11-15T14:16:09,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/520d7a608e5a4c269ad89eefa7b1f718 is 50, key is test_row_0/C:col10/1731680167756/Put/seqid=0 2024-11-15T14:16:09,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:09,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742170_1346 (size=9657) 2024-11-15T14:16:09,440 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T14:16:09,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:09,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:09,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:09,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:09,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:09,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/520d7a608e5a4c269ad89eefa7b1f718 2024-11-15T14:16:09,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/03c064cd6d594b9aac1e0f8370cf6e28 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/03c064cd6d594b9aac1e0f8370cf6e28 2024-11-15T14:16:09,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/03c064cd6d594b9aac1e0f8370cf6e28, entries=200, sequenceid=14, filesize=38.6 K 2024-11-15T14:16:09,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/0bf6989de027496485ef5f1b1101ee13 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/0bf6989de027496485ef5f1b1101ee13 2024-11-15T14:16:09,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/0bf6989de027496485ef5f1b1101ee13, entries=100, sequenceid=14, filesize=9.4 K 2024-11-15T14:16:09,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/520d7a608e5a4c269ad89eefa7b1f718 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/520d7a608e5a4c269ad89eefa7b1f718 2024-11-15T14:16:09,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/520d7a608e5a4c269ad89eefa7b1f718, entries=100, sequenceid=14, filesize=9.4 K 2024-11-15T14:16:09,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:09,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:09,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:09,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:09,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a6c59e08ce89fd58e535f1c83edb91e8 in 2076ms, sequenceid=14, compaction requested=false 2024-11-15T14:16:09,835 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T14:16:09,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:09,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T14:16:09,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T14:16:09,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:09,992 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-15T14:16:09,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:09,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:09,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:09,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:09,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:09,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:10,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a6e0ee26198f41738d896de8f8016d2f_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680167897/Put/seqid=0 2024-11-15T14:16:10,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742171_1347 (size=12154) 2024-11-15T14:16:10,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:10,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680230166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680230167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680230181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680230181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680230184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680230294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680230294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680230304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680230304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680230312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:10,462 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a6e0ee26198f41738d896de8f8016d2f_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a6e0ee26198f41738d896de8f8016d2f_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:10,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/4e8bc4fb4a164bf99fdd8577a7fb131f, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:10,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/4e8bc4fb4a164bf99fdd8577a7fb131f is 175, key is test_row_0/A:col10/1731680167897/Put/seqid=0 2024-11-15T14:16:10,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742172_1348 (size=30955) 2024-11-15T14:16:10,501 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/4e8bc4fb4a164bf99fdd8577a7fb131f 2024-11-15T14:16:10,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680230517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680230517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680230517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680230517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/a7b631b8fc5d49efa3b48f5600ed323f is 50, key is test_row_0/B:col10/1731680167897/Put/seqid=0 2024-11-15T14:16:10,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680230536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742173_1349 (size=12001) 2024-11-15T14:16:10,565 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/a7b631b8fc5d49efa3b48f5600ed323f 2024-11-15T14:16:10,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/efa8b675c25a44448b64bbbe97c2aec1 is 50, key is test_row_0/C:col10/1731680167897/Put/seqid=0 2024-11-15T14:16:10,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742174_1350 (size=12001) 2024-11-15T14:16:10,716 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/efa8b675c25a44448b64bbbe97c2aec1 2024-11-15T14:16:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/4e8bc4fb4a164bf99fdd8577a7fb131f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/4e8bc4fb4a164bf99fdd8577a7fb131f 2024-11-15T14:16:10,748 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/4e8bc4fb4a164bf99fdd8577a7fb131f, entries=150, sequenceid=50, filesize=30.2 K 2024-11-15T14:16:10,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/a7b631b8fc5d49efa3b48f5600ed323f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/a7b631b8fc5d49efa3b48f5600ed323f 2024-11-15T14:16:10,784 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/a7b631b8fc5d49efa3b48f5600ed323f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T14:16:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/efa8b675c25a44448b64bbbe97c2aec1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/efa8b675c25a44448b64bbbe97c2aec1 2024-11-15T14:16:10,789 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/efa8b675c25a44448b64bbbe97c2aec1, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T14:16:10,791 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a6c59e08ce89fd58e535f1c83edb91e8 in 798ms, sequenceid=50, compaction requested=false 2024-11-15T14:16:10,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:10,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:10,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-15T14:16:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-15T14:16:10,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-15T14:16:10,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9860 sec 2024-11-15T14:16:10,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.0050 sec 2024-11-15T14:16:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:10,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T14:16:10,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:10,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:10,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:10,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:10,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:10,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:10,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157c75fdef670045a892c6ce3d433c6136_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680170162/Put/seqid=0 2024-11-15T14:16:10,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742175_1351 (size=17034) 2024-11-15T14:16:10,920 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:10,934 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157c75fdef670045a892c6ce3d433c6136_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c75fdef670045a892c6ce3d433c6136_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:10,935 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/589aa28041ba49079a8e9dbe768ae2da, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:10,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/589aa28041ba49079a8e9dbe768ae2da is 175, key is test_row_0/A:col10/1731680170162/Put/seqid=0 2024-11-15T14:16:10,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680230937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680230949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680230956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680230957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:10,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680230976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742176_1352 (size=48139) 2024-11-15T14:16:11,009 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/589aa28041ba49079a8e9dbe768ae2da 2024-11-15T14:16:11,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/6da36d8b5a44415f8ddefc542f68588d is 50, key is test_row_0/B:col10/1731680170162/Put/seqid=0 2024-11-15T14:16:11,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680231082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680231083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680231088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680231088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742177_1353 (size=12001) 2024-11-15T14:16:11,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/6da36d8b5a44415f8ddefc542f68588d 2024-11-15T14:16:11,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680231103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/7c697ce2211946a882740baf3b1dc4ea is 50, key is test_row_0/C:col10/1731680170162/Put/seqid=0 2024-11-15T14:16:11,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742178_1354 (size=12001) 2024-11-15T14:16:11,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680231313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680231314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680231326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680231332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680231339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/7c697ce2211946a882740baf3b1dc4ea 2024-11-15T14:16:11,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680231632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/589aa28041ba49079a8e9dbe768ae2da as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/589aa28041ba49079a8e9dbe768ae2da 2024-11-15T14:16:11,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/589aa28041ba49079a8e9dbe768ae2da, entries=250, sequenceid=62, filesize=47.0 K 2024-11-15T14:16:11,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680231645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/6da36d8b5a44415f8ddefc542f68588d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/6da36d8b5a44415f8ddefc542f68588d 2024-11-15T14:16:11,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680231652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680231652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/6da36d8b5a44415f8ddefc542f68588d, entries=150, sequenceid=62, filesize=11.7 K 2024-11-15T14:16:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680231652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:11,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/7c697ce2211946a882740baf3b1dc4ea as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/7c697ce2211946a882740baf3b1dc4ea 2024-11-15T14:16:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:11,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/7c697ce2211946a882740baf3b1dc4ea, entries=150, sequenceid=62, filesize=11.7 K 2024-11-15T14:16:11,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a6c59e08ce89fd58e535f1c83edb91e8 in 844ms, sequenceid=62, compaction requested=true 2024-11-15T14:16:11,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:11,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:11,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:11,683 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:11,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:11,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:11,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:11,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:16:11,685 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:11,689 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:11,689 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118639 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:11,689 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/B is initiating minor compaction (all files) 2024-11-15T14:16:11,689 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/A is initiating minor compaction (all files) 2024-11-15T14:16:11,689 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/A in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:11,689 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/B in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:11,689 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/03c064cd6d594b9aac1e0f8370cf6e28, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/4e8bc4fb4a164bf99fdd8577a7fb131f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/589aa28041ba49079a8e9dbe768ae2da] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=115.9 K 2024-11-15T14:16:11,689 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:11,689 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/0bf6989de027496485ef5f1b1101ee13, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/a7b631b8fc5d49efa3b48f5600ed323f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/6da36d8b5a44415f8ddefc542f68588d] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=32.9 K 2024-11-15T14:16:11,689 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/03c064cd6d594b9aac1e0f8370cf6e28, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/4e8bc4fb4a164bf99fdd8577a7fb131f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/589aa28041ba49079a8e9dbe768ae2da] 2024-11-15T14:16:11,690 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03c064cd6d594b9aac1e0f8370cf6e28, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731680167749 2024-11-15T14:16:11,690 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bf6989de027496485ef5f1b1101ee13, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731680167755 2024-11-15T14:16:11,690 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e8bc4fb4a164bf99fdd8577a7fb131f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731680167895 2024-11-15T14:16:11,690 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting a7b631b8fc5d49efa3b48f5600ed323f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731680167895 2024-11-15T14:16:11,693 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 589aa28041ba49079a8e9dbe768ae2da, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731680170162 2024-11-15T14:16:11,693 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 6da36d8b5a44415f8ddefc542f68588d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731680170162 2024-11-15T14:16:11,713 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#B#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:11,715 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:11,717 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/1398ddab04ba40b8ac3e48ceaf6bf1d3 is 50, key is test_row_0/B:col10/1731680170162/Put/seqid=0 2024-11-15T14:16:11,727 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411151fab77c7c134405b9d58f2aec639dc58_a6c59e08ce89fd58e535f1c83edb91e8 store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:11,730 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411151fab77c7c134405b9d58f2aec639dc58_a6c59e08ce89fd58e535f1c83edb91e8, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:11,730 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151fab77c7c134405b9d58f2aec639dc58_a6c59e08ce89fd58e535f1c83edb91e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:11,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742179_1355 (size=12104) 2024-11-15T14:16:11,792 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/1398ddab04ba40b8ac3e48ceaf6bf1d3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/1398ddab04ba40b8ac3e48ceaf6bf1d3 2024-11-15T14:16:11,807 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/B of a6c59e08ce89fd58e535f1c83edb91e8 into 1398ddab04ba40b8ac3e48ceaf6bf1d3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:11,807 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:11,807 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/B, priority=13, startTime=1731680171683; duration=0sec 2024-11-15T14:16:11,807 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:11,807 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:B 2024-11-15T14:16:11,807 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:11,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742180_1356 (size=4469) 2024-11-15T14:16:11,817 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:11,818 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/C is initiating minor compaction (all files) 2024-11-15T14:16:11,818 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/C in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:11,818 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/520d7a608e5a4c269ad89eefa7b1f718, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/efa8b675c25a44448b64bbbe97c2aec1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/7c697ce2211946a882740baf3b1dc4ea] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=32.9 K 2024-11-15T14:16:11,818 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 520d7a608e5a4c269ad89eefa7b1f718, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731680167755 2024-11-15T14:16:11,824 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting efa8b675c25a44448b64bbbe97c2aec1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731680167895 2024-11-15T14:16:11,824 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#A#compaction#306 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:11,824 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c697ce2211946a882740baf3b1dc4ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731680170162 2024-11-15T14:16:11,825 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/ba33dc3c4fd347d98cd7350b0c6b6f54 is 175, key is test_row_0/A:col10/1731680170162/Put/seqid=0 2024-11-15T14:16:11,851 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#C#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:11,852 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/2b3872e5a97b4f4e828d748430388d63 is 50, key is test_row_0/C:col10/1731680170162/Put/seqid=0 2024-11-15T14:16:11,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742181_1357 (size=31058) 2024-11-15T14:16:11,900 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/ba33dc3c4fd347d98cd7350b0c6b6f54 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/ba33dc3c4fd347d98cd7350b0c6b6f54 2024-11-15T14:16:11,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742182_1358 (size=12104) 2024-11-15T14:16:11,916 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/A of a6c59e08ce89fd58e535f1c83edb91e8 into ba33dc3c4fd347d98cd7350b0c6b6f54(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:11,916 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:11,916 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/A, priority=13, startTime=1731680171683; duration=0sec 2024-11-15T14:16:11,916 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:11,916 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:A 2024-11-15T14:16:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T14:16:11,992 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:11,992 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-15T14:16:11,994 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:11,996 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:11,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T14:16:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T14:16:12,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T14:16:12,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:12,156 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T14:16:12,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:12,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:12,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:12,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:12,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:12,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:12,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:12,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151f3fd66dec634de0a2aee0d2ed2f3b9e_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680170948/Put/seqid=0 2024-11-15T14:16:12,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680232212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680232215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680232219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742183_1359 (size=12154) 2024-11-15T14:16:12,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:12,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680232235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,261 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151f3fd66dec634de0a2aee0d2ed2f3b9e_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151f3fd66dec634de0a2aee0d2ed2f3b9e_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:12,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/459b7b7a1dc5453ea6b1c0be5a32cb3c, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:12,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/459b7b7a1dc5453ea6b1c0be5a32cb3c is 175, key is test_row_0/A:col10/1731680170948/Put/seqid=0 2024-11-15T14:16:12,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680232241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742184_1360 (size=30955) 2024-11-15T14:16:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T14:16:12,325 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/2b3872e5a97b4f4e828d748430388d63 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/2b3872e5a97b4f4e828d748430388d63 2024-11-15T14:16:12,331 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/C of a6c59e08ce89fd58e535f1c83edb91e8 into 2b3872e5a97b4f4e828d748430388d63(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:12,331 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:12,331 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/C, priority=13, startTime=1731680171683; duration=0sec 2024-11-15T14:16:12,332 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:12,332 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:C 2024-11-15T14:16:12,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680232342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680232342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680232352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680232361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680232382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680232571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680232571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680232582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680232592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680232601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T14:16:12,705 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/459b7b7a1dc5453ea6b1c0be5a32cb3c 2024-11-15T14:16:12,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/9289591552194a069447cf1024618786 is 50, key is test_row_0/B:col10/1731680170948/Put/seqid=0 2024-11-15T14:16:12,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742185_1361 (size=12001) 2024-11-15T14:16:12,741 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/9289591552194a069447cf1024618786 2024-11-15T14:16:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/76091b99cc2c4714b4c1ed25cd348352 is 50, key is test_row_0/C:col10/1731680170948/Put/seqid=0 2024-11-15T14:16:12,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742186_1362 (size=12001) 2024-11-15T14:16:12,796 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/76091b99cc2c4714b4c1ed25cd348352 2024-11-15T14:16:12,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/459b7b7a1dc5453ea6b1c0be5a32cb3c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/459b7b7a1dc5453ea6b1c0be5a32cb3c 2024-11-15T14:16:12,817 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/459b7b7a1dc5453ea6b1c0be5a32cb3c, entries=150, sequenceid=84, filesize=30.2 K 2024-11-15T14:16:12,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/9289591552194a069447cf1024618786 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/9289591552194a069447cf1024618786 2024-11-15T14:16:12,840 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/9289591552194a069447cf1024618786, entries=150, sequenceid=84, filesize=11.7 K 2024-11-15T14:16:12,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/76091b99cc2c4714b4c1ed25cd348352 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/76091b99cc2c4714b4c1ed25cd348352 2024-11-15T14:16:12,855 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/76091b99cc2c4714b4c1ed25cd348352, entries=150, sequenceid=84, filesize=11.7 K 2024-11-15T14:16:12,856 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a6c59e08ce89fd58e535f1c83edb91e8 in 700ms, sequenceid=84, compaction requested=false 2024-11-15T14:16:12,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:12,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:12,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-15T14:16:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-15T14:16:12,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-15T14:16:12,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 862 msec 2024-11-15T14:16:12,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 870 msec 2024-11-15T14:16:12,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T14:16:12,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:12,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:12,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:12,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:12,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:12,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:12,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:12,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152e77a54520ca43449eba0a159eda33f8_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680172910/Put/seqid=0 2024-11-15T14:16:12,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742187_1363 (size=14594) 2024-11-15T14:16:13,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680233006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680233020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680233020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680233022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680233025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T14:16:13,132 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:13,133 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:13,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-15T14:16:13,135 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:13,136 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:13,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T14:16:13,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680233132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680233152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680233152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680233162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680233165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T14:16:13,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T14:16:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:13,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,379 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:13,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680233365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680233372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680233372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680233392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,408 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152e77a54520ca43449eba0a159eda33f8_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152e77a54520ca43449eba0a159eda33f8_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:13,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680233395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,416 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/6481f08b23dc4a9a96fdaba32e418644, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:13,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/6481f08b23dc4a9a96fdaba32e418644 is 175, key is test_row_0/A:col10/1731680172910/Put/seqid=0 2024-11-15T14:16:13,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T14:16:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T14:16:13,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:13,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:13,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:13,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742188_1364 (size=39549) 2024-11-15T14:16:13,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T14:16:13,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:13,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:13,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:13,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680233692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680233692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680233702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680233712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680233723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:13,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T14:16:13,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T14:16:13,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:13,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:13,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:13,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,875 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/6481f08b23dc4a9a96fdaba32e418644 2024-11-15T14:16:13,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/acdafd629f2d44df851d2a507b08f4e2 is 50, key is test_row_0/B:col10/1731680172910/Put/seqid=0 2024-11-15T14:16:13,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T14:16:13,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:13,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:13,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:13,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:13,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742189_1365 (size=12001) 2024-11-15T14:16:14,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T14:16:14,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:14,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:14,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:14,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:14,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:14,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680234212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:14,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680234213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:14,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680234213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:14,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680234233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:14,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T14:16:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:14,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680234253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:14,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T14:16:14,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/acdafd629f2d44df851d2a507b08f4e2 2024-11-15T14:16:14,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/02377d5dc18546038192b4f5f7c4249a is 50, key is test_row_0/C:col10/1731680172910/Put/seqid=0 2024-11-15T14:16:14,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742190_1366 (size=12001) 2024-11-15T14:16:14,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/02377d5dc18546038192b4f5f7c4249a 2024-11-15T14:16:14,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T14:16:14,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/6481f08b23dc4a9a96fdaba32e418644 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/6481f08b23dc4a9a96fdaba32e418644 2024-11-15T14:16:14,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:14,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:14,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:14,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:14,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:14,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/6481f08b23dc4a9a96fdaba32e418644, entries=200, sequenceid=98, filesize=38.6 K 2024-11-15T14:16:14,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/acdafd629f2d44df851d2a507b08f4e2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/acdafd629f2d44df851d2a507b08f4e2 2024-11-15T14:16:14,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/acdafd629f2d44df851d2a507b08f4e2, entries=150, sequenceid=98, filesize=11.7 K 2024-11-15T14:16:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/02377d5dc18546038192b4f5f7c4249a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/02377d5dc18546038192b4f5f7c4249a 2024-11-15T14:16:14,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/02377d5dc18546038192b4f5f7c4249a, entries=150, sequenceid=98, filesize=11.7 K 2024-11-15T14:16:14,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a6c59e08ce89fd58e535f1c83edb91e8 in 1534ms, sequenceid=98, compaction requested=true 2024-11-15T14:16:14,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:14,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:14,450 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:14,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:14,450 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:14,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:14,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:14,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:14,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:14,451 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:14,451 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/A is initiating minor compaction (all files) 2024-11-15T14:16:14,451 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/A in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:14,451 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/ba33dc3c4fd347d98cd7350b0c6b6f54, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/459b7b7a1dc5453ea6b1c0be5a32cb3c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/6481f08b23dc4a9a96fdaba32e418644] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=99.2 K 2024-11-15T14:16:14,451 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:14,451 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/ba33dc3c4fd347d98cd7350b0c6b6f54, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/459b7b7a1dc5453ea6b1c0be5a32cb3c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/6481f08b23dc4a9a96fdaba32e418644] 2024-11-15T14:16:14,452 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:14,452 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/B is initiating minor compaction (all files) 2024-11-15T14:16:14,452 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/B in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:14,452 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/1398ddab04ba40b8ac3e48ceaf6bf1d3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/9289591552194a069447cf1024618786, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/acdafd629f2d44df851d2a507b08f4e2] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=35.3 K 2024-11-15T14:16:14,452 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba33dc3c4fd347d98cd7350b0c6b6f54, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731680170162 2024-11-15T14:16:14,452 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1398ddab04ba40b8ac3e48ceaf6bf1d3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731680170162 2024-11-15T14:16:14,452 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 459b7b7a1dc5453ea6b1c0be5a32cb3c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731680170948 2024-11-15T14:16:14,453 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6481f08b23dc4a9a96fdaba32e418644, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731680172213 2024-11-15T14:16:14,453 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9289591552194a069447cf1024618786, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731680170948 2024-11-15T14:16:14,453 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting acdafd629f2d44df851d2a507b08f4e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731680172213 2024-11-15T14:16:14,471 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:14,473 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#B#compaction#314 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:14,473 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/c1ee17400dfb4ddd9ae1470a9aefac90 is 50, key is test_row_0/B:col10/1731680172910/Put/seqid=0 2024-11-15T14:16:14,477 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111559771b4744364c85b03598012b015770_a6c59e08ce89fd58e535f1c83edb91e8 store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:14,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742191_1367 (size=12207) 2024-11-15T14:16:14,479 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111559771b4744364c85b03598012b015770_a6c59e08ce89fd58e535f1c83edb91e8, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:14,480 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111559771b4744364c85b03598012b015770_a6c59e08ce89fd58e535f1c83edb91e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:14,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742192_1368 (size=4469) 2024-11-15T14:16:14,494 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/c1ee17400dfb4ddd9ae1470a9aefac90 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c1ee17400dfb4ddd9ae1470a9aefac90 2024-11-15T14:16:14,494 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#A#compaction#315 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:14,495 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/45aea5716f8941af8ff7f9d6a01aa5ff is 175, key is test_row_0/A:col10/1731680172910/Put/seqid=0 2024-11-15T14:16:14,505 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/B of a6c59e08ce89fd58e535f1c83edb91e8 into c1ee17400dfb4ddd9ae1470a9aefac90(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:14,505 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:14,505 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/B, priority=13, startTime=1731680174450; duration=0sec 2024-11-15T14:16:14,505 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:14,505 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:B 2024-11-15T14:16:14,505 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:14,510 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:14,510 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/C is initiating minor compaction (all files) 2024-11-15T14:16:14,510 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/C in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:14,511 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/2b3872e5a97b4f4e828d748430388d63, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/76091b99cc2c4714b4c1ed25cd348352, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/02377d5dc18546038192b4f5f7c4249a] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=35.3 K 2024-11-15T14:16:14,512 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b3872e5a97b4f4e828d748430388d63, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731680170162 2024-11-15T14:16:14,512 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 76091b99cc2c4714b4c1ed25cd348352, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731680170948 2024-11-15T14:16:14,512 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 02377d5dc18546038192b4f5f7c4249a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731680172213 2024-11-15T14:16:14,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742193_1369 (size=31161) 2024-11-15T14:16:14,537 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#C#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:14,538 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/b4214c8341d747efbd17c8d526d5e326 is 50, key is test_row_0/C:col10/1731680172910/Put/seqid=0 2024-11-15T14:16:14,548 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/45aea5716f8941af8ff7f9d6a01aa5ff as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/45aea5716f8941af8ff7f9d6a01aa5ff 2024-11-15T14:16:14,554 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/A of a6c59e08ce89fd58e535f1c83edb91e8 into 45aea5716f8941af8ff7f9d6a01aa5ff(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:14,554 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:14,554 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/A, priority=13, startTime=1731680174450; duration=0sec 2024-11-15T14:16:14,554 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:14,554 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:A 2024-11-15T14:16:14,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T14:16:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:14,580 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T14:16:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:14,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:14,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742194_1370 (size=12207) 2024-11-15T14:16:14,609 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/b4214c8341d747efbd17c8d526d5e326 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/b4214c8341d747efbd17c8d526d5e326 2024-11-15T14:16:14,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111569c8b891e0714a89ba4f8502c8a5c151_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680173011/Put/seqid=0 2024-11-15T14:16:14,618 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/C of a6c59e08ce89fd58e535f1c83edb91e8 into b4214c8341d747efbd17c8d526d5e326(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:14,618 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:14,618 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/C, priority=13, startTime=1731680174450; duration=0sec 2024-11-15T14:16:14,618 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:14,618 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:C 2024-11-15T14:16:14,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742195_1371 (size=12154) 2024-11-15T14:16:14,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:14,648 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111569c8b891e0714a89ba4f8502c8a5c151_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111569c8b891e0714a89ba4f8502c8a5c151_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/68bf2268e2e14f4eae940e94a6a836a7, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/68bf2268e2e14f4eae940e94a6a836a7 is 175, key is test_row_0/A:col10/1731680173011/Put/seqid=0 2024-11-15T14:16:14,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742196_1372 (size=30955) 2024-11-15T14:16:15,112 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/68bf2268e2e14f4eae940e94a6a836a7 2024-11-15T14:16:15,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/c9ac51259ac04544b02f17275c53b48d is 50, key is test_row_0/B:col10/1731680173011/Put/seqid=0 2024-11-15T14:16:15,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742197_1373 (size=12001) 2024-11-15T14:16:15,219 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/c9ac51259ac04544b02f17275c53b48d 2024-11-15T14:16:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/29a84ad777c243cca92f516478280c80 is 50, key is test_row_0/C:col10/1731680173011/Put/seqid=0 2024-11-15T14:16:15,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:15,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742198_1374 (size=12001) 2024-11-15T14:16:15,277 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/29a84ad777c243cca92f516478280c80 2024-11-15T14:16:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T14:16:15,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680235275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680235276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680235279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680235295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/68bf2268e2e14f4eae940e94a6a836a7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/68bf2268e2e14f4eae940e94a6a836a7 2024-11-15T14:16:15,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680235296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,326 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/68bf2268e2e14f4eae940e94a6a836a7, entries=150, sequenceid=120, filesize=30.2 K 2024-11-15T14:16:15,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/c9ac51259ac04544b02f17275c53b48d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c9ac51259ac04544b02f17275c53b48d 2024-11-15T14:16:15,346 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c9ac51259ac04544b02f17275c53b48d, entries=150, sequenceid=120, filesize=11.7 K 2024-11-15T14:16:15,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/29a84ad777c243cca92f516478280c80 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/29a84ad777c243cca92f516478280c80 2024-11-15T14:16:15,362 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/29a84ad777c243cca92f516478280c80, entries=150, sequenceid=120, filesize=11.7 K 2024-11-15T14:16:15,367 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a6c59e08ce89fd58e535f1c83edb91e8 in 787ms, sequenceid=120, compaction requested=false 2024-11-15T14:16:15,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:15,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:15,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-15T14:16:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-15T14:16:15,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-15T14:16:15,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2330 sec 2024-11-15T14:16:15,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.2390 sec 2024-11-15T14:16:15,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T14:16:15,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:15,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:15,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:15,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:15,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:15,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:15,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a84d7b7f7ab8426797176346625c2c2e_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680175241/Put/seqid=0 2024-11-15T14:16:15,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742199_1375 (size=14694) 2024-11-15T14:16:15,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680235493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680235505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680235506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680235507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680235512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680235612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680235632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680235632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680235643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680235643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680235833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680235852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680235852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680235852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680235852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:15,911 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:15,921 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a84d7b7f7ab8426797176346625c2c2e_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a84d7b7f7ab8426797176346625c2c2e_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:15,926 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/907563f9567b49779ef8cb6ccd5a5633, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:15,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/907563f9567b49779ef8cb6ccd5a5633 is 175, key is test_row_0/A:col10/1731680175241/Put/seqid=0 2024-11-15T14:16:16,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680236142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680236175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680236175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680236182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680236183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742200_1376 (size=39649) 2024-11-15T14:16:16,231 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/907563f9567b49779ef8cb6ccd5a5633 2024-11-15T14:16:16,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/5f1c98de8b564bce858be674dad8be40 is 50, key is test_row_0/B:col10/1731680175241/Put/seqid=0 2024-11-15T14:16:16,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742201_1377 (size=12051) 2024-11-15T14:16:16,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/5f1c98de8b564bce858be674dad8be40 2024-11-15T14:16:16,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/5ed1e6288bc04b65a5a2f8207c839129 is 50, key is test_row_0/C:col10/1731680175241/Put/seqid=0 2024-11-15T14:16:16,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742202_1378 (size=12051) 2024-11-15T14:16:16,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/5ed1e6288bc04b65a5a2f8207c839129 2024-11-15T14:16:16,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/907563f9567b49779ef8cb6ccd5a5633 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/907563f9567b49779ef8cb6ccd5a5633 2024-11-15T14:16:16,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/907563f9567b49779ef8cb6ccd5a5633, entries=200, sequenceid=134, filesize=38.7 K 2024-11-15T14:16:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/5f1c98de8b564bce858be674dad8be40 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5f1c98de8b564bce858be674dad8be40 2024-11-15T14:16:16,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5f1c98de8b564bce858be674dad8be40, entries=150, sequenceid=134, filesize=11.8 K 2024-11-15T14:16:16,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/5ed1e6288bc04b65a5a2f8207c839129 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/5ed1e6288bc04b65a5a2f8207c839129 2024-11-15T14:16:16,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/5ed1e6288bc04b65a5a2f8207c839129, entries=150, sequenceid=134, filesize=11.8 K 2024-11-15T14:16:16,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a6c59e08ce89fd58e535f1c83edb91e8 in 1086ms, sequenceid=134, compaction requested=true 2024-11-15T14:16:16,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:16,496 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:16,497 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:16,498 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:16,498 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:16,498 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/B is initiating minor compaction (all files) 2024-11-15T14:16:16,498 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/A is initiating minor compaction (all files) 2024-11-15T14:16:16,498 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/B in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:16,498 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/A in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:16,499 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/45aea5716f8941af8ff7f9d6a01aa5ff, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/68bf2268e2e14f4eae940e94a6a836a7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/907563f9567b49779ef8cb6ccd5a5633] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=99.4 K 2024-11-15T14:16:16,499 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c1ee17400dfb4ddd9ae1470a9aefac90, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c9ac51259ac04544b02f17275c53b48d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5f1c98de8b564bce858be674dad8be40] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=35.4 K 2024-11-15T14:16:16,499 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:16,499 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/45aea5716f8941af8ff7f9d6a01aa5ff, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/68bf2268e2e14f4eae940e94a6a836a7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/907563f9567b49779ef8cb6ccd5a5633] 2024-11-15T14:16:16,499 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45aea5716f8941af8ff7f9d6a01aa5ff, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731680172213 2024-11-15T14:16:16,499 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c1ee17400dfb4ddd9ae1470a9aefac90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731680172213 2024-11-15T14:16:16,499 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c9ac51259ac04544b02f17275c53b48d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731680173011 2024-11-15T14:16:16,499 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68bf2268e2e14f4eae940e94a6a836a7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731680173011 2024-11-15T14:16:16,500 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 907563f9567b49779ef8cb6ccd5a5633, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731680175241 2024-11-15T14:16:16,500 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f1c98de8b564bce858be674dad8be40, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731680175241 2024-11-15T14:16:16,519 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:16,531 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:16,531 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/eb2f366e9bc945e49acf4d5c906e760a is 50, key is test_row_0/B:col10/1731680175241/Put/seqid=0 2024-11-15T14:16:16,544 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111533f74d0dbf104cc282f0efc823c36693_a6c59e08ce89fd58e535f1c83edb91e8 store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:16,546 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111533f74d0dbf104cc282f0efc823c36693_a6c59e08ce89fd58e535f1c83edb91e8, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:16,547 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111533f74d0dbf104cc282f0efc823c36693_a6c59e08ce89fd58e535f1c83edb91e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:16,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742203_1379 (size=12359) 2024-11-15T14:16:16,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742204_1380 (size=4469) 2024-11-15T14:16:16,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T14:16:16,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:16,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:16,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:16,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:16,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:16,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:16,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e116bda9c2e1422e84dccda98a33d07b_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680175485/Put/seqid=0 2024-11-15T14:16:16,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742205_1381 (size=14794) 2024-11-15T14:16:16,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680236732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680236734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680236756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680236757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680236759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680236861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680236872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680236882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680236892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680236892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:16,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:16,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,021 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/eb2f366e9bc945e49acf4d5c906e760a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/eb2f366e9bc945e49acf4d5c906e760a 2024-11-15T14:16:17,022 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#A#compaction#323 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:17,023 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/fd30a11e7a44420783557dac5a6fe92b is 175, key is test_row_0/A:col10/1731680175241/Put/seqid=0 2024-11-15T14:16:17,047 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/B of a6c59e08ce89fd58e535f1c83edb91e8 into eb2f366e9bc945e49acf4d5c906e760a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:17,047 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:17,047 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/B, priority=13, startTime=1731680176497; duration=0sec 2024-11-15T14:16:17,047 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:17,047 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:B 2024-11-15T14:16:17,047 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:17,048 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:17,048 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/C is initiating minor compaction (all files) 2024-11-15T14:16:17,048 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/C in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:17,049 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/b4214c8341d747efbd17c8d526d5e326, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/29a84ad777c243cca92f516478280c80, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/5ed1e6288bc04b65a5a2f8207c839129] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=35.4 K 2024-11-15T14:16:17,049 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting b4214c8341d747efbd17c8d526d5e326, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731680172213 2024-11-15T14:16:17,050 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 29a84ad777c243cca92f516478280c80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731680173011 2024-11-15T14:16:17,050 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed1e6288bc04b65a5a2f8207c839129, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731680175241 2024-11-15T14:16:17,069 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#C#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:17,069 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/77fb4d4ef4be4fb0befffdb7750ecef6 is 50, key is test_row_0/C:col10/1731680175241/Put/seqid=0 2024-11-15T14:16:17,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742206_1382 (size=31313) 2024-11-15T14:16:17,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680237094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742207_1383 (size=12359) 2024-11-15T14:16:17,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680237112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680237122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680237132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,143 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:17,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680237134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,161 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e116bda9c2e1422e84dccda98a33d07b_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e116bda9c2e1422e84dccda98a33d07b_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:17,163 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/c23a4673412d475c80583916723e1015, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:17,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/c23a4673412d475c80583916723e1015 is 175, key is test_row_0/A:col10/1731680175485/Put/seqid=0 2024-11-15T14:16:17,164 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/77fb4d4ef4be4fb0befffdb7750ecef6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/77fb4d4ef4be4fb0befffdb7750ecef6 2024-11-15T14:16:17,169 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/C of a6c59e08ce89fd58e535f1c83edb91e8 into 77fb4d4ef4be4fb0befffdb7750ecef6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:17,169 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:17,169 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/C, priority=13, startTime=1731680176497; duration=0sec 2024-11-15T14:16:17,169 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:17,169 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:C 2024-11-15T14:16:17,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742208_1384 (size=39749) 2024-11-15T14:16:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T14:16:17,302 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:17,302 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-15T14:16:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T14:16:17,304 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:17,305 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:17,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T14:16:17,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680237422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680237435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680237443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680237443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T14:16:17,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:17,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:17,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:17,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:17,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:17,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:17,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680237462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,490 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/fd30a11e7a44420783557dac5a6fe92b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/fd30a11e7a44420783557dac5a6fe92b 2024-11-15T14:16:17,496 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/A of a6c59e08ce89fd58e535f1c83edb91e8 into fd30a11e7a44420783557dac5a6fe92b(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:17,496 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:17,496 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/A, priority=13, startTime=1731680176496; duration=0sec 2024-11-15T14:16:17,496 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:17,496 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:A 2024-11-15T14:16:17,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T14:16:17,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:17,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:17,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:17,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:17,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T14:16:17,638 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/c23a4673412d475c80583916723e1015 2024-11-15T14:16:17,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/5b87236d4d1144a28d3bf5a57003fc2d is 50, key is test_row_0/B:col10/1731680175485/Put/seqid=0 2024-11-15T14:16:17,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742209_1385 (size=12151) 2024-11-15T14:16:17,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T14:16:17,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:17,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:17,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:17,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:17,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:17,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T14:16:17,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:17,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:17,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:17,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:17,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T14:16:17,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680237953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680237953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680237962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680237962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:17,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680237982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:18,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T14:16:18,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:18,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:18,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:18,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:18,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:18,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/5b87236d4d1144a28d3bf5a57003fc2d 2024-11-15T14:16:18,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/08b1cb41325c4197835b4eb6e0589b98 is 50, key is test_row_0/C:col10/1731680175485/Put/seqid=0 2024-11-15T14:16:18,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742210_1386 (size=12151) 2024-11-15T14:16:18,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/08b1cb41325c4197835b4eb6e0589b98 2024-11-15T14:16:18,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/c23a4673412d475c80583916723e1015 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/c23a4673412d475c80583916723e1015 2024-11-15T14:16:18,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T14:16:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:18,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:18,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/c23a4673412d475c80583916723e1015, entries=200, sequenceid=154, filesize=38.8 K 2024-11-15T14:16:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:18,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/5b87236d4d1144a28d3bf5a57003fc2d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5b87236d4d1144a28d3bf5a57003fc2d 2024-11-15T14:16:18,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5b87236d4d1144a28d3bf5a57003fc2d, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T14:16:18,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/08b1cb41325c4197835b4eb6e0589b98 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/08b1cb41325c4197835b4eb6e0589b98 2024-11-15T14:16:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/08b1cb41325c4197835b4eb6e0589b98, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T14:16:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a6c59e08ce89fd58e535f1c83edb91e8 in 1636ms, sequenceid=154, compaction requested=false 2024-11-15T14:16:18,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T14:16:18,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:18,400 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T14:16:18,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:18,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:18,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:18,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:18,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a4fb4ddeab124921a971c5754870e2b5_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680176741/Put/seqid=0 2024-11-15T14:16:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T14:16:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742211_1387 (size=9814) 2024-11-15T14:16:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,505 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a4fb4ddeab124921a971c5754870e2b5_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a4fb4ddeab124921a971c5754870e2b5_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:18,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/35e8d143e6c547c180125a63bb8c7d07, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:18,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/35e8d143e6c547c180125a63bb8c7d07 is 175, key is test_row_0/A:col10/1731680176741/Put/seqid=0 2024-11-15T14:16:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742212_1388 (size=22461) 2024-11-15T14:16:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,955 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/35e8d143e6c547c180125a63bb8c7d07 2024-11-15T14:16:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/54853d1f880b4d80b69e5e1ca8ee4269 is 50, key is test_row_0/B:col10/1731680176741/Put/seqid=0 2024-11-15T14:16:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742213_1389 (size=9757) 2024-11-15T14:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,026 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/54853d1f880b4d80b69e5e1ca8ee4269 2024-11-15T14:16:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:19,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/2665cd52a25d4b508d72e075cecc8366 is 50, key is test_row_0/C:col10/1731680176741/Put/seqid=0 2024-11-15T14:16:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742214_1390 (size=9757) 2024-11-15T14:16:19,115 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/2665cd52a25d4b508d72e075cecc8366 2024-11-15T14:16:19,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/35e8d143e6c547c180125a63bb8c7d07 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/35e8d143e6c547c180125a63bb8c7d07 2024-11-15T14:16:19,134 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/35e8d143e6c547c180125a63bb8c7d07, entries=100, sequenceid=169, filesize=21.9 K 2024-11-15T14:16:19,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/54853d1f880b4d80b69e5e1ca8ee4269 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/54853d1f880b4d80b69e5e1ca8ee4269 2024-11-15T14:16:19,141 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/54853d1f880b4d80b69e5e1ca8ee4269, entries=100, sequenceid=169, filesize=9.5 K 2024-11-15T14:16:19,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/2665cd52a25d4b508d72e075cecc8366 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/2665cd52a25d4b508d72e075cecc8366 2024-11-15T14:16:19,148 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/2665cd52a25d4b508d72e075cecc8366, entries=100, sequenceid=169, filesize=9.5 K 2024-11-15T14:16:19,150 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=100.63 KB/103050 for a6c59e08ce89fd58e535f1c83edb91e8 in 751ms, sequenceid=169, compaction requested=true 2024-11-15T14:16:19,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:19,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:19,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-15T14:16:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-15T14:16:19,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-15T14:16:19,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8560 sec 2024-11-15T14:16:19,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.8630 sec 2024-11-15T14:16:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:19,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T14:16:19,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:19,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:19,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:19,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:19,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:19,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:19,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153c29796597c746368a837cccee73a152_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680179176/Put/seqid=0 2024-11-15T14:16:19,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680239231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680239232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680239234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680239254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680239255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742215_1391 (size=27248) 2024-11-15T14:16:19,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680239372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680239374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680239375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680239402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680239402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T14:16:19,452 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:19,453 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-15T14:16:19,455 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T14:16:19,456 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:19,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T14:16:19,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T14:16:19,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:19,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:19,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:19,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:19,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:19,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680239615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680239616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680239616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680239633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680239633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,714 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:19,720 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153c29796597c746368a837cccee73a152_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153c29796597c746368a837cccee73a152_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:19,721 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/f86a86924830480a8bba8a1809490fd4, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:19,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/f86a86924830480a8bba8a1809490fd4 is 175, key is test_row_0/A:col10/1731680179176/Put/seqid=0 2024-11-15T14:16:19,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742216_1392 (size=83035) 2024-11-15T14:16:19,728 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/f86a86924830480a8bba8a1809490fd4 2024-11-15T14:16:19,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/333bed2f402c4b628757347c5be0589e is 50, key is test_row_0/B:col10/1731680179176/Put/seqid=0 2024-11-15T14:16:19,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T14:16:19,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:19,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:19,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:19,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:19,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T14:16:19,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742217_1393 (size=12151) 2024-11-15T14:16:19,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T14:16:19,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:19,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:19,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:19,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:19,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:19,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680239933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680239942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680239942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680239961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680239961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T14:16:20,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:20,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:20,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:20,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:20,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T14:16:20,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/333bed2f402c4b628757347c5be0589e 2024-11-15T14:16:20,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/794a704b499f4cafb91e0cc324f66e7b is 50, key is test_row_0/C:col10/1731680179176/Put/seqid=0 2024-11-15T14:16:20,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T14:16:20,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:20,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:20,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:20,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:20,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:20,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742218_1394 (size=12151) 2024-11-15T14:16:20,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/794a704b499f4cafb91e0cc324f66e7b 2024-11-15T14:16:20,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/f86a86924830480a8bba8a1809490fd4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f86a86924830480a8bba8a1809490fd4 2024-11-15T14:16:20,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f86a86924830480a8bba8a1809490fd4, entries=450, sequenceid=188, filesize=81.1 K 2024-11-15T14:16:20,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/333bed2f402c4b628757347c5be0589e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/333bed2f402c4b628757347c5be0589e 2024-11-15T14:16:20,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/333bed2f402c4b628757347c5be0589e, entries=150, sequenceid=188, filesize=11.9 K 2024-11-15T14:16:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/794a704b499f4cafb91e0cc324f66e7b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/794a704b499f4cafb91e0cc324f66e7b 2024-11-15T14:16:20,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/794a704b499f4cafb91e0cc324f66e7b, entries=150, sequenceid=188, filesize=11.9 K 2024-11-15T14:16:20,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a6c59e08ce89fd58e535f1c83edb91e8 in 1124ms, sequenceid=188, compaction requested=true 2024-11-15T14:16:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:20,302 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:16:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:16:20,303 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:16:20,305 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 176558 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:16:20,306 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/A is initiating minor compaction (all files) 2024-11-15T14:16:20,306 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/A in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:20,306 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/fd30a11e7a44420783557dac5a6fe92b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/c23a4673412d475c80583916723e1015, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/35e8d143e6c547c180125a63bb8c7d07, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f86a86924830480a8bba8a1809490fd4] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=172.4 K 2024-11-15T14:16:20,306 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46418 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:16:20,306 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:20,306 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/B is initiating minor compaction (all files) 2024-11-15T14:16:20,306 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/fd30a11e7a44420783557dac5a6fe92b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/c23a4673412d475c80583916723e1015, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/35e8d143e6c547c180125a63bb8c7d07, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f86a86924830480a8bba8a1809490fd4] 2024-11-15T14:16:20,306 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/B in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:20,306 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/eb2f366e9bc945e49acf4d5c906e760a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5b87236d4d1144a28d3bf5a57003fc2d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/54853d1f880b4d80b69e5e1ca8ee4269, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/333bed2f402c4b628757347c5be0589e] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=45.3 K 2024-11-15T14:16:20,306 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd30a11e7a44420783557dac5a6fe92b, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731680175241 2024-11-15T14:16:20,307 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting eb2f366e9bc945e49acf4d5c906e760a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731680175241 2024-11-15T14:16:20,307 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c23a4673412d475c80583916723e1015, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731680175485 2024-11-15T14:16:20,307 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b87236d4d1144a28d3bf5a57003fc2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731680175485 2024-11-15T14:16:20,307 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35e8d143e6c547c180125a63bb8c7d07, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731680176741 2024-11-15T14:16:20,307 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 54853d1f880b4d80b69e5e1ca8ee4269, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731680176741 2024-11-15T14:16:20,307 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f86a86924830480a8bba8a1809490fd4, keycount=450, bloomtype=ROW, size=81.1 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731680179060 2024-11-15T14:16:20,308 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 333bed2f402c4b628757347c5be0589e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731680179126 2024-11-15T14:16:20,320 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:20,323 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#B#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:20,324 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/5a68e1f2d4bc4ab99db5faa387408297 is 50, key is test_row_0/B:col10/1731680179176/Put/seqid=0 2024-11-15T14:16:20,326 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411159f21057372e940e6a16c7a00dd515017_a6c59e08ce89fd58e535f1c83edb91e8 store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:20,330 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411159f21057372e940e6a16c7a00dd515017_a6c59e08ce89fd58e535f1c83edb91e8, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:20,331 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159f21057372e940e6a16c7a00dd515017_a6c59e08ce89fd58e535f1c83edb91e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:20,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742220_1396 (size=4469) 2024-11-15T14:16:20,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T14:16:20,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:20,394 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T14:16:20,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:20,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:20,395 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#A#compaction#335 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:20,395 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/b6d04846e64d4f08889559fd86ee8e08 is 175, key is test_row_0/A:col10/1731680179176/Put/seqid=0 2024-11-15T14:16:20,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742219_1395 (size=12595) 2024-11-15T14:16:20,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f4d01da7b0d34de7875be221f06f4fc1_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680179249/Put/seqid=0 2024-11-15T14:16:20,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742221_1397 (size=31549) 2024-11-15T14:16:20,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:20,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742222_1398 (size=12304) 2024-11-15T14:16:20,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680240516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680240519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680240525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680240527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680240539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T14:16:20,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680240642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680240651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680240652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680240662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680240672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,825 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/5a68e1f2d4bc4ab99db5faa387408297 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5a68e1f2d4bc4ab99db5faa387408297 2024-11-15T14:16:20,831 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/B of a6c59e08ce89fd58e535f1c83edb91e8 into 5a68e1f2d4bc4ab99db5faa387408297(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:20,831 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:20,831 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/B, priority=12, startTime=1731680180302; duration=0sec 2024-11-15T14:16:20,831 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:20,831 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:B 2024-11-15T14:16:20,831 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:16:20,835 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46418 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:16:20,835 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/C is initiating minor compaction (all files) 2024-11-15T14:16:20,835 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/C in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:20,836 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/77fb4d4ef4be4fb0befffdb7750ecef6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/08b1cb41325c4197835b4eb6e0589b98, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/2665cd52a25d4b508d72e075cecc8366, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/794a704b499f4cafb91e0cc324f66e7b] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=45.3 K 2024-11-15T14:16:20,836 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 77fb4d4ef4be4fb0befffdb7750ecef6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731680175241 2024-11-15T14:16:20,836 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 08b1cb41325c4197835b4eb6e0589b98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731680175485 2024-11-15T14:16:20,837 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2665cd52a25d4b508d72e075cecc8366, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731680176741 2024-11-15T14:16:20,840 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 794a704b499f4cafb91e0cc324f66e7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731680179126 2024-11-15T14:16:20,864 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#C#compaction#338 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:20,864 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/9444b4bf7a84456ea6d063cddc26b445 is 50, key is test_row_0/C:col10/1731680179176/Put/seqid=0 2024-11-15T14:16:20,871 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/b6d04846e64d4f08889559fd86ee8e08 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/b6d04846e64d4f08889559fd86ee8e08 2024-11-15T14:16:20,878 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/A of a6c59e08ce89fd58e535f1c83edb91e8 into b6d04846e64d4f08889559fd86ee8e08(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:20,878 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:20,878 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/A, priority=12, startTime=1731680180302; duration=0sec 2024-11-15T14:16:20,878 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:20,878 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:A 2024-11-15T14:16:20,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:20,887 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f4d01da7b0d34de7875be221f06f4fc1_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f4d01da7b0d34de7875be221f06f4fc1_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:20,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/3fb0bb5b14134db28674743355af5fcb, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:20,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/3fb0bb5b14134db28674743355af5fcb is 175, key is test_row_0/A:col10/1731680179249/Put/seqid=0 2024-11-15T14:16:20,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680240872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742223_1399 (size=12595) 2024-11-15T14:16:20,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680240881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680240882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680240892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680240912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:20,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742224_1400 (size=31105) 2024-11-15T14:16:20,944 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/3fb0bb5b14134db28674743355af5fcb 2024-11-15T14:16:20,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/c3426dde38334d59b2242c06b2b295ea is 50, key is test_row_0/B:col10/1731680179249/Put/seqid=0 2024-11-15T14:16:21,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742225_1401 (size=12151) 2024-11-15T14:16:21,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680241201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680241225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680241225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680241225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680241225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,317 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/9444b4bf7a84456ea6d063cddc26b445 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/9444b4bf7a84456ea6d063cddc26b445 2024-11-15T14:16:21,330 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/C of a6c59e08ce89fd58e535f1c83edb91e8 into 9444b4bf7a84456ea6d063cddc26b445(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:21,330 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:21,330 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/C, priority=12, startTime=1731680180302; duration=0sec 2024-11-15T14:16:21,330 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:21,345 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:C 2024-11-15T14:16:21,425 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/c3426dde38334d59b2242c06b2b295ea 2024-11-15T14:16:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/b098f53f31114293b23b9e24a568bcc1 is 50, key is test_row_0/C:col10/1731680179249/Put/seqid=0 2024-11-15T14:16:21,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742226_1402 (size=12151) 2024-11-15T14:16:21,500 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/b098f53f31114293b23b9e24a568bcc1 2024-11-15T14:16:21,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/3fb0bb5b14134db28674743355af5fcb as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/3fb0bb5b14134db28674743355af5fcb 2024-11-15T14:16:21,523 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/3fb0bb5b14134db28674743355af5fcb, entries=150, sequenceid=201, filesize=30.4 K 2024-11-15T14:16:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/c3426dde38334d59b2242c06b2b295ea as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c3426dde38334d59b2242c06b2b295ea 2024-11-15T14:16:21,532 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c3426dde38334d59b2242c06b2b295ea, entries=150, sequenceid=201, filesize=11.9 K 2024-11-15T14:16:21,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/b098f53f31114293b23b9e24a568bcc1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/b098f53f31114293b23b9e24a568bcc1 2024-11-15T14:16:21,544 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/b098f53f31114293b23b9e24a568bcc1, entries=150, sequenceid=201, filesize=11.9 K 2024-11-15T14:16:21,546 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a6c59e08ce89fd58e535f1c83edb91e8 in 1151ms, sequenceid=201, compaction requested=false 2024-11-15T14:16:21,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:21,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:21,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-15T14:16:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-15T14:16:21,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-15T14:16:21,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0910 sec 2024-11-15T14:16:21,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.0970 sec 2024-11-15T14:16:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T14:16:21,602 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:21,602 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-15T14:16:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T14:16:21,604 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:21,605 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:21,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T14:16:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:21,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T14:16:21,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:21,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:21,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:21,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:21,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:21,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:21,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:21,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:21,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:21,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:21,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:21,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:21,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155f845048b99f44bb85cef8f7a92b97c3_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680180515/Put/seqid=0 2024-11-15T14:16:21,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680241774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680241773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742227_1403 (size=14794) 2024-11-15T14:16:21,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680241786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680241792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680241792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:21,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:21,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:21,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:21,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:21,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:21,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680241902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680241902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T14:16:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680241912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680241912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:21,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680241922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:22,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680242121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680242121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680242134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680242134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680242143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,200 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,209 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155f845048b99f44bb85cef8f7a92b97c3_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155f845048b99f44bb85cef8f7a92b97c3_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:22,210 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/5951ea7fe2f14d9391cd69b33b2314af, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:22,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/5951ea7fe2f14d9391cd69b33b2314af is 175, key is test_row_0/A:col10/1731680180515/Put/seqid=0 2024-11-15T14:16:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T14:16:22,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742228_1404 (size=39749) 2024-11-15T14:16:22,254 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/5951ea7fe2f14d9391cd69b33b2314af 2024-11-15T14:16:22,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/11445411584c4988806755c4651ff89d is 50, key is test_row_0/B:col10/1731680180515/Put/seqid=0 2024-11-15T14:16:22,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742229_1405 (size=12151) 2024-11-15T14:16:22,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/11445411584c4988806755c4651ff89d 2024-11-15T14:16:22,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/a93271113b474450bf177128ef1c7288 is 50, key is test_row_0/C:col10/1731680180515/Put/seqid=0 2024-11-15T14:16:22,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:22,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:22,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742230_1406 (size=12151) 2024-11-15T14:16:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/a93271113b474450bf177128ef1c7288 2024-11-15T14:16:22,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/5951ea7fe2f14d9391cd69b33b2314af as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5951ea7fe2f14d9391cd69b33b2314af 2024-11-15T14:16:22,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5951ea7fe2f14d9391cd69b33b2314af, entries=200, sequenceid=224, filesize=38.8 K 2024-11-15T14:16:22,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/11445411584c4988806755c4651ff89d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/11445411584c4988806755c4651ff89d 2024-11-15T14:16:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/11445411584c4988806755c4651ff89d, entries=150, sequenceid=224, filesize=11.9 K 2024-11-15T14:16:22,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/a93271113b474450bf177128ef1c7288 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/a93271113b474450bf177128ef1c7288 2024-11-15T14:16:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/a93271113b474450bf177128ef1c7288, entries=150, sequenceid=224, filesize=11.9 K 2024-11-15T14:16:22,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a6c59e08ce89fd58e535f1c83edb91e8 in 692ms, sequenceid=224, compaction requested=true 2024-11-15T14:16:22,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:22,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:22,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:22,438 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:22,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:22,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:22,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:22,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:16:22,438 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,452 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,452 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/A is initiating minor compaction (all files) 2024-11-15T14:16:22,452 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/A in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,452 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/b6d04846e64d4f08889559fd86ee8e08, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/3fb0bb5b14134db28674743355af5fcb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5951ea7fe2f14d9391cd69b33b2314af] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=100.0 K 2024-11-15T14:16:22,452 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,452 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/b6d04846e64d4f08889559fd86ee8e08, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/3fb0bb5b14134db28674743355af5fcb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5951ea7fe2f14d9391cd69b33b2314af] 2024-11-15T14:16:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,458 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:22,459 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/B is initiating minor compaction (all files) 2024-11-15T14:16:22,459 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/B in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,459 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5a68e1f2d4bc4ab99db5faa387408297, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c3426dde38334d59b2242c06b2b295ea, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/11445411584c4988806755c4651ff89d] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=36.0 K 2024-11-15T14:16:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,459 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6d04846e64d4f08889559fd86ee8e08, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731680179126 2024-11-15T14:16:22,460 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fb0bb5b14134db28674743355af5fcb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731680179230 2024-11-15T14:16:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,460 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5951ea7fe2f14d9391cd69b33b2314af, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731680180515 2024-11-15T14:16:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,462 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a68e1f2d4bc4ab99db5faa387408297, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731680179126 2024-11-15T14:16:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,467 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c3426dde38334d59b2242c06b2b295ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731680179230 2024-11-15T14:16:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,470 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 11445411584c4988806755c4651ff89d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731680180515 2024-11-15T14:16:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,479 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,485 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,485 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/a169a5788c6e4ea995077a3be47ab5b3 is 50, key is test_row_0/B:col10/1731680180515/Put/seqid=0 2024-11-15T14:16:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,499 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115f4226bd315fd4497a1cdd6081581e241_a6c59e08ce89fd58e535f1c83edb91e8 store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:22,501 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115f4226bd315fd4497a1cdd6081581e241_a6c59e08ce89fd58e535f1c83edb91e8, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:22,501 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f4226bd315fd4497a1cdd6081581e241_a6c59e08ce89fd58e535f1c83edb91e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T14:16:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:22,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:22,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:22,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:22,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:22,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:22,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742231_1407 (size=12697) 2024-11-15T14:16:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742232_1408 (size=4469) 2024-11-15T14:16:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b09412a803c14b8e8f53b151fb61f2c0_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_1/A:col10/1731680181786/Put/seqid=0 2024-11-15T14:16:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,552 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#A#compaction#344 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:22,552 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/622f7bfe8ca64534bdfd751bc1dceac3 is 175, key is test_row_0/A:col10/1731680180515/Put/seqid=0 2024-11-15T14:16:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742233_1409 (size=12304) 2024-11-15T14:16:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742234_1410 (size=31651) 2024-11-15T14:16:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,613 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/622f7bfe8ca64534bdfd751bc1dceac3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/622f7bfe8ca64534bdfd751bc1dceac3 2024-11-15T14:16:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,620 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/A of a6c59e08ce89fd58e535f1c83edb91e8 into 622f7bfe8ca64534bdfd751bc1dceac3(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:22,620 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:22,620 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/A, priority=13, startTime=1731680182438; duration=0sec 2024-11-15T14:16:22,620 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:22,621 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:A 2024-11-15T14:16:22,621 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,621 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:22,622 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/C is initiating minor compaction (all files) 2024-11-15T14:16:22,622 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/C in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,622 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/9444b4bf7a84456ea6d063cddc26b445, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/b098f53f31114293b23b9e24a568bcc1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/a93271113b474450bf177128ef1c7288] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=36.0 K 2024-11-15T14:16:22,622 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9444b4bf7a84456ea6d063cddc26b445, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731680179126 2024-11-15T14:16:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,622 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b098f53f31114293b23b9e24a568bcc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731680179230 2024-11-15T14:16:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,623 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a93271113b474450bf177128ef1c7288, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731680180515 2024-11-15T14:16:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,633 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#C#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:22,633 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/c5c36c9b93ba4ba9aeaa23826fa9797f is 50, key is test_row_0/C:col10/1731680180515/Put/seqid=0 2024-11-15T14:16:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:22,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742235_1411 (size=12697) 2024-11-15T14:16:22,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680242654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680242655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680242656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680242673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680242686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T14:16:22,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680242783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680242783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680242783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680242801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680242802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:22,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:22,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:22,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:22,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:22,939 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/a169a5788c6e4ea995077a3be47ab5b3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/a169a5788c6e4ea995077a3be47ab5b3 2024-11-15T14:16:22,945 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/B of a6c59e08ce89fd58e535f1c83edb91e8 into a169a5788c6e4ea995077a3be47ab5b3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:22,945 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:22,945 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/B, priority=13, startTime=1731680182438; duration=0sec 2024-11-15T14:16:22,945 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:22,945 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:B 2024-11-15T14:16:23,003 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:23,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680242992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680242992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680243002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:23,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:23,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,020 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b09412a803c14b8e8f53b151fb61f2c0_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b09412a803c14b8e8f53b151fb61f2c0_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:23,022 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/f3f8931124774db29baf665c3f825599, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:23,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/f3f8931124774db29baf665c3f825599 is 175, key is test_row_1/A:col10/1731680181786/Put/seqid=0 2024-11-15T14:16:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680243022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680243022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,061 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/c5c36c9b93ba4ba9aeaa23826fa9797f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/c5c36c9b93ba4ba9aeaa23826fa9797f 2024-11-15T14:16:23,070 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/C of a6c59e08ce89fd58e535f1c83edb91e8 into c5c36c9b93ba4ba9aeaa23826fa9797f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:23,070 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:23,070 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/C, priority=13, startTime=1731680182438; duration=0sec 2024-11-15T14:16:23,070 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:23,070 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:C 2024-11-15T14:16:23,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742236_1412 (size=31101) 2024-11-15T14:16:23,077 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/f3f8931124774db29baf665c3f825599 2024-11-15T14:16:23,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/dcb297757347487b83f87adcbf0afedf is 50, key is test_row_1/B:col10/1731680181786/Put/seqid=0 2024-11-15T14:16:23,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742237_1413 (size=9757) 2024-11-15T14:16:23,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/dcb297757347487b83f87adcbf0afedf 2024-11-15T14:16:23,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/49edd05e2b7d48a39c48f4b08b525506 is 50, key is test_row_1/C:col10/1731680181786/Put/seqid=0 2024-11-15T14:16:23,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:23,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:23,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742238_1414 (size=9757) 2024-11-15T14:16:23,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/49edd05e2b7d48a39c48f4b08b525506 2024-11-15T14:16:23,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/f3f8931124774db29baf665c3f825599 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f3f8931124774db29baf665c3f825599 2024-11-15T14:16:23,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f3f8931124774db29baf665c3f825599, entries=150, sequenceid=237, filesize=30.4 K 2024-11-15T14:16:23,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/dcb297757347487b83f87adcbf0afedf as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/dcb297757347487b83f87adcbf0afedf 2024-11-15T14:16:23,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/dcb297757347487b83f87adcbf0afedf, entries=100, sequenceid=237, filesize=9.5 K 2024-11-15T14:16:23,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/49edd05e2b7d48a39c48f4b08b525506 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/49edd05e2b7d48a39c48f4b08b525506 2024-11-15T14:16:23,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/49edd05e2b7d48a39c48f4b08b525506, entries=100, sequenceid=237, filesize=9.5 K 2024-11-15T14:16:23,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a6c59e08ce89fd58e535f1c83edb91e8 in 749ms, sequenceid=237, compaction requested=false 2024-11-15T14:16:23,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:23,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680182652, max timestamp=1731680183317Num uniques 859; 2024-11-15T14:16:23,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:16:23,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680182652, max timestamp=1731680183317Num uniques 859; 2024-11-15T14:16:23,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:16:23,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680182652, max timestamp=1731680183317Num uniques 859; 2024-11-15T14:16:23,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:16:23,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-15T14:16:23,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:23,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:23,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:23,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:23,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:23,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:23,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:23,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:23,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159a17cd75f4574b4aae809aba6b53d340_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680183341/Put/seqid=0 2024-11-15T14:16:23,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742239_1415 (size=17484) 2024-11-15T14:16:23,412 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:23,418 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159a17cd75f4574b4aae809aba6b53d340_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159a17cd75f4574b4aae809aba6b53d340_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:23,419 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/90f7dbf143554af9b882df75f9da1ee2, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:23,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/90f7dbf143554af9b882df75f9da1ee2 is 175, key is test_row_0/A:col10/1731680183341/Put/seqid=0 2024-11-15T14:16:23,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680183413, max timestamp=1731680183450Num uniques 859; 2024-11-15T14:16:23,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:16:23,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680183413, max timestamp=1731680183453Num uniques 859; 2024-11-15T14:16:23,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:16:23,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742240_1416 (size=48589) 2024-11-15T14:16:23,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680183413, max timestamp=1731680183453Num uniques 859; 2024-11-15T14:16:23,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:16:23,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:23,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680243534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680243548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680243555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680243584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680243587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680243695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680243695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680243695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680243722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680243722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T14:16:23,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:23,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:23,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,861 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/90f7dbf143554af9b882df75f9da1ee2 2024-11-15T14:16:23,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/3c56a486cc79483da107ec2ff2ad5435 is 50, key is test_row_0/B:col10/1731680183341/Put/seqid=0 2024-11-15T14:16:23,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742241_1417 (size=12251) 2024-11-15T14:16:23,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/3c56a486cc79483da107ec2ff2ad5435 2024-11-15T14:16:23,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/dc393122b0584055b78fcc2ec9db6308 is 50, key is test_row_0/C:col10/1731680183341/Put/seqid=0 2024-11-15T14:16:23,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680243933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680243934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680243939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680243942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680243951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:23,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:23,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:23,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:24,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742242_1418 (size=12251) 2024-11-15T14:16:24,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:24,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:24,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:24,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:24,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:24,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:24,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680244272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680244272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680244272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680244282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680244282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:24,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:24,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:24,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:24,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:24,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:24,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/dc393122b0584055b78fcc2ec9db6308 2024-11-15T14:16:24,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/90f7dbf143554af9b882df75f9da1ee2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/90f7dbf143554af9b882df75f9da1ee2 2024-11-15T14:16:24,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/90f7dbf143554af9b882df75f9da1ee2, entries=250, sequenceid=265, filesize=47.5 K 2024-11-15T14:16:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/3c56a486cc79483da107ec2ff2ad5435 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/3c56a486cc79483da107ec2ff2ad5435 2024-11-15T14:16:24,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:24,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:24,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:24,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:24,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:24,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:24,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/3c56a486cc79483da107ec2ff2ad5435, entries=150, sequenceid=265, filesize=12.0 K 2024-11-15T14:16:24,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/dc393122b0584055b78fcc2ec9db6308 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/dc393122b0584055b78fcc2ec9db6308 2024-11-15T14:16:24,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/dc393122b0584055b78fcc2ec9db6308, entries=150, sequenceid=265, filesize=12.0 K 2024-11-15T14:16:24,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=140.89 KB/144270 for a6c59e08ce89fd58e535f1c83edb91e8 in 1123ms, sequenceid=265, compaction requested=true 2024-11-15T14:16:24,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:24,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:24,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:24,467 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:24,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:24,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:24,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:24,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:16:24,467 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:24,468 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:24,469 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/B is initiating minor compaction (all files) 2024-11-15T14:16:24,469 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/B in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:24,469 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/a169a5788c6e4ea995077a3be47ab5b3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/dcb297757347487b83f87adcbf0afedf, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/3c56a486cc79483da107ec2ff2ad5435] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=33.9 K 2024-11-15T14:16:24,469 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111341 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:24,469 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting a169a5788c6e4ea995077a3be47ab5b3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731680180515 2024-11-15T14:16:24,469 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/A is initiating minor compaction (all files) 2024-11-15T14:16:24,469 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/A in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:24,469 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/622f7bfe8ca64534bdfd751bc1dceac3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f3f8931124774db29baf665c3f825599, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/90f7dbf143554af9b882df75f9da1ee2] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=108.7 K 2024-11-15T14:16:24,469 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:24,469 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/622f7bfe8ca64534bdfd751bc1dceac3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f3f8931124774db29baf665c3f825599, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/90f7dbf143554af9b882df75f9da1ee2] 2024-11-15T14:16:24,472 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb297757347487b83f87adcbf0afedf, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731680181786 2024-11-15T14:16:24,472 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 622f7bfe8ca64534bdfd751bc1dceac3, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731680180515 2024-11-15T14:16:24,477 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c56a486cc79483da107ec2ff2ad5435, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731680182654 2024-11-15T14:16:24,477 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3f8931124774db29baf665c3f825599, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731680181786 2024-11-15T14:16:24,478 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90f7dbf143554af9b882df75f9da1ee2, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731680182654 2024-11-15T14:16:24,498 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:24,509 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:24,509 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/8f61716f0c51464195bf645769337f27 is 50, key is test_row_0/B:col10/1731680183341/Put/seqid=0 2024-11-15T14:16:24,534 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115fe5dd328931342888a90f883d11e3a85_a6c59e08ce89fd58e535f1c83edb91e8 store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:24,538 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115fe5dd328931342888a90f883d11e3a85_a6c59e08ce89fd58e535f1c83edb91e8, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:24,540 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fe5dd328931342888a90f883d11e3a85_a6c59e08ce89fd58e535f1c83edb91e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:24,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742243_1419 (size=12899) 2024-11-15T14:16:24,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742244_1420 (size=4469) 2024-11-15T14:16:24,557 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#A#compaction#353 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:24,558 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/fdeda0a0176849df82e38c9aaba48e51 is 175, key is test_row_0/A:col10/1731680183341/Put/seqid=0 2024-11-15T14:16:24,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742245_1421 (size=31853) 2024-11-15T14:16:24,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T14:16:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:24,609 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-15T14:16:24,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:24,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:24,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:24,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:24,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:24,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:24,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115636658d9c43b447b83e9162f877c5096_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680183574/Put/seqid=0 2024-11-15T14:16:24,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742246_1422 (size=12454) 2024-11-15T14:16:24,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:24,657 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115636658d9c43b447b83e9162f877c5096_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115636658d9c43b447b83e9162f877c5096_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:24,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/105020222b814b248296e656d4dceff7, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:24,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/105020222b814b248296e656d4dceff7 is 175, key is test_row_0/A:col10/1731680183574/Put/seqid=0 2024-11-15T14:16:24,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742247_1423 (size=31255) 2024-11-15T14:16:24,705 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/105020222b814b248296e656d4dceff7 2024-11-15T14:16:24,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/b6bd24e38fad4abf91e6744f56c3425b is 50, key is test_row_0/B:col10/1731680183574/Put/seqid=0 2024-11-15T14:16:24,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742248_1424 (size=12301) 2024-11-15T14:16:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:24,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:24,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680244814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680244816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680244817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680244818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680244823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680244932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,962 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/8f61716f0c51464195bf645769337f27 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/8f61716f0c51464195bf645769337f27 2024-11-15T14:16:24,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680244952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680244952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680244953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,967 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/B of a6c59e08ce89fd58e535f1c83edb91e8 into 8f61716f0c51464195bf645769337f27(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:24,967 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:24,968 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/B, priority=13, startTime=1731680184467; duration=0sec 2024-11-15T14:16:24,968 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:24,968 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:B 2024-11-15T14:16:24,968 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:24,969 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:24,969 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/C is initiating minor compaction (all files) 2024-11-15T14:16:24,969 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/C in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:24,969 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/c5c36c9b93ba4ba9aeaa23826fa9797f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/49edd05e2b7d48a39c48f4b08b525506, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/dc393122b0584055b78fcc2ec9db6308] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=33.9 K 2024-11-15T14:16:24,969 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c5c36c9b93ba4ba9aeaa23826fa9797f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731680180515 2024-11-15T14:16:24,970 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 49edd05e2b7d48a39c48f4b08b525506, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731680181786 2024-11-15T14:16:24,970 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting dc393122b0584055b78fcc2ec9db6308, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731680182654 2024-11-15T14:16:24,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680244962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:24,985 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#C#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:24,985 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/e0d5ff24efbf4681a3475b3e75cc0bfd is 50, key is test_row_0/C:col10/1731680183341/Put/seqid=0 2024-11-15T14:16:25,014 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/fdeda0a0176849df82e38c9aaba48e51 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/fdeda0a0176849df82e38c9aaba48e51 2024-11-15T14:16:25,021 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/A of a6c59e08ce89fd58e535f1c83edb91e8 into fdeda0a0176849df82e38c9aaba48e51(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:25,021 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:25,021 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/A, priority=13, startTime=1731680184467; duration=0sec 2024-11-15T14:16:25,021 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:25,021 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:A 2024-11-15T14:16:25,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742249_1425 (size=12899) 2024-11-15T14:16:25,033 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/e0d5ff24efbf4681a3475b3e75cc0bfd as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/e0d5ff24efbf4681a3475b3e75cc0bfd 2024-11-15T14:16:25,039 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/C of a6c59e08ce89fd58e535f1c83edb91e8 into e0d5ff24efbf4681a3475b3e75cc0bfd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:25,039 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:25,039 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/C, priority=13, startTime=1731680184467; duration=0sec 2024-11-15T14:16:25,039 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:25,039 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:C 2024-11-15T14:16:25,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680245152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,158 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/b6bd24e38fad4abf91e6744f56c3425b 2024-11-15T14:16:25,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/c18720776e584d3cbe3c97c22abea83c is 50, key is test_row_0/C:col10/1731680183574/Put/seqid=0 2024-11-15T14:16:25,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680245172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680245172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680245172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680245191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742250_1426 (size=12301) 2024-11-15T14:16:25,196 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/c18720776e584d3cbe3c97c22abea83c 2024-11-15T14:16:25,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/105020222b814b248296e656d4dceff7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/105020222b814b248296e656d4dceff7 2024-11-15T14:16:25,211 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/105020222b814b248296e656d4dceff7, entries=150, sequenceid=302, filesize=30.5 K 2024-11-15T14:16:25,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/b6bd24e38fad4abf91e6744f56c3425b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/b6bd24e38fad4abf91e6744f56c3425b 2024-11-15T14:16:25,217 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/b6bd24e38fad4abf91e6744f56c3425b, entries=150, sequenceid=302, filesize=12.0 K 2024-11-15T14:16:25,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/c18720776e584d3cbe3c97c22abea83c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/c18720776e584d3cbe3c97c22abea83c 2024-11-15T14:16:25,226 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/c18720776e584d3cbe3c97c22abea83c, entries=150, sequenceid=302, filesize=12.0 K 2024-11-15T14:16:25,227 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for a6c59e08ce89fd58e535f1c83edb91e8 in 618ms, sequenceid=302, compaction requested=false 2024-11-15T14:16:25,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:25,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:25,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-15T14:16:25,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-15T14:16:25,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-15T14:16:25,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6240 sec 2024-11-15T14:16:25,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 3.6290 sec 2024-11-15T14:16:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:25,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T14:16:25,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:25,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:25,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:25,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:25,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:25,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:25,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f78ef9524e2a4403956eae86058dd39e_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680184821/Put/seqid=0 2024-11-15T14:16:25,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742251_1427 (size=14994) 2024-11-15T14:16:25,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680185552, max timestamp=1731680185570Num uniques 859; 2024-11-15T14:16:25,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680185552, max timestamp=1731680185570Num uniques 859; 2024-11-15T14:16:25,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680185552, max timestamp=1731680185570Num uniques 859; 2024-11-15T14:16:25,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:16:25,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:16:25,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:16:25,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680245646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680245645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680245647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680245648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680245666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T14:16:25,774 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:25,774 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-15T14:16:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T14:16:25,776 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:25,777 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:25,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:25,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680245774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680245774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680245775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680245782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680245782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T14:16:25,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T14:16:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:25,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:25,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:25,932 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:25,940 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f78ef9524e2a4403956eae86058dd39e_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f78ef9524e2a4403956eae86058dd39e_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:25,944 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/5ff830a6f7b64040976bc11be5789178, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:25,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/5ff830a6f7b64040976bc11be5789178 is 175, key is test_row_0/A:col10/1731680184821/Put/seqid=0 2024-11-15T14:16:25,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742252_1428 (size=39949) 2024-11-15T14:16:26,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680246002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680246002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680246002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680246002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680246002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T14:16:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:26,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T14:16:26,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T14:16:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:26,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680246322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680246322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680246322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680246322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680246322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,356 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/5ff830a6f7b64040976bc11be5789178 2024-11-15T14:16:26,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/43c437bb69524c0eae83a3cf33e297bd is 50, key is test_row_0/B:col10/1731680184821/Put/seqid=0 2024-11-15T14:16:26,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742253_1429 (size=12301) 2024-11-15T14:16:26,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T14:16:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:26,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T14:16:26,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T14:16:26,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:26,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:26,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:26,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T14:16:26,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:26,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:26,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:26,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/43c437bb69524c0eae83a3cf33e297bd 2024-11-15T14:16:26,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/d6e7b24af5b14e46a753c7afdc0de5f0 is 50, key is test_row_0/C:col10/1731680184821/Put/seqid=0 2024-11-15T14:16:26,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742254_1430 (size=12301) 2024-11-15T14:16:26,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45208 deadline: 1731680246841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45250 deadline: 1731680246841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45290 deadline: 1731680246841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45218 deadline: 1731680246842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45284 deadline: 1731680246842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:16:26,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T14:16:26,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:26,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:26,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:26,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T14:16:27,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T14:16:27,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:27,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:27,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:27,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:27,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:27,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T14:16:27,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:27,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:27,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/d6e7b24af5b14e46a753c7afdc0de5f0 2024-11-15T14:16:27,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/5ff830a6f7b64040976bc11be5789178 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5ff830a6f7b64040976bc11be5789178 2024-11-15T14:16:27,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5ff830a6f7b64040976bc11be5789178, entries=200, sequenceid=315, filesize=39.0 K 2024-11-15T14:16:27,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/43c437bb69524c0eae83a3cf33e297bd as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/43c437bb69524c0eae83a3cf33e297bd 2024-11-15T14:16:27,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/43c437bb69524c0eae83a3cf33e297bd, entries=150, sequenceid=315, filesize=12.0 K 2024-11-15T14:16:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/d6e7b24af5b14e46a753c7afdc0de5f0 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/d6e7b24af5b14e46a753c7afdc0de5f0 2024-11-15T14:16:27,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/d6e7b24af5b14e46a753c7afdc0de5f0, entries=150, sequenceid=315, filesize=12.0 K 2024-11-15T14:16:27,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a6c59e08ce89fd58e535f1c83edb91e8 in 1720ms, sequenceid=315, compaction requested=true 2024-11-15T14:16:27,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:27,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:27,190 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:27,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:27,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:27,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:27,190 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:27,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6c59e08ce89fd58e535f1c83edb91e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:27,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:27,190 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103057 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:27,190 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:27,190 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/A is initiating minor compaction (all files) 2024-11-15T14:16:27,190 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/B is initiating minor compaction (all files) 2024-11-15T14:16:27,190 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/A in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:27,190 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/B in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:27,191 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/fdeda0a0176849df82e38c9aaba48e51, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/105020222b814b248296e656d4dceff7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5ff830a6f7b64040976bc11be5789178] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=100.6 K 2024-11-15T14:16:27,191 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/8f61716f0c51464195bf645769337f27, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/b6bd24e38fad4abf91e6744f56c3425b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/43c437bb69524c0eae83a3cf33e297bd] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=36.6 K 2024-11-15T14:16:27,191 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:27,191 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/fdeda0a0176849df82e38c9aaba48e51, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/105020222b814b248296e656d4dceff7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5ff830a6f7b64040976bc11be5789178] 2024-11-15T14:16:27,191 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f61716f0c51464195bf645769337f27, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731680182654 2024-11-15T14:16:27,191 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdeda0a0176849df82e38c9aaba48e51, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731680182654 2024-11-15T14:16:27,191 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting b6bd24e38fad4abf91e6744f56c3425b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731680183521 2024-11-15T14:16:27,191 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 105020222b814b248296e656d4dceff7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731680183521 2024-11-15T14:16:27,191 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 43c437bb69524c0eae83a3cf33e297bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731680184817 2024-11-15T14:16:27,192 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ff830a6f7b64040976bc11be5789178, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731680184816 2024-11-15T14:16:27,201 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#B#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:27,201 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/ff485fafbff646d6ba7781e1009c660c is 50, key is test_row_0/B:col10/1731680184821/Put/seqid=0 2024-11-15T14:16:27,202 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:27,214 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115351cf07f3a6844dfb1b41d915b5bb05c_a6c59e08ce89fd58e535f1c83edb91e8 store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:27,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742255_1431 (size=13051) 2024-11-15T14:16:27,215 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115351cf07f3a6844dfb1b41d915b5bb05c_a6c59e08ce89fd58e535f1c83edb91e8, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:27,215 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115351cf07f3a6844dfb1b41d915b5bb05c_a6c59e08ce89fd58e535f1c83edb91e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:27,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742256_1432 (size=4469) 2024-11-15T14:16:27,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T14:16:27,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:27,312 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-15T14:16:27,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:27,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:27,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:27,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:27,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:27,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:27,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158ccfedb58ec841b6b3579b277aaab465_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680185645/Put/seqid=0 2024-11-15T14:16:27,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742257_1433 (size=12454) 2024-11-15T14:16:27,618 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/ff485fafbff646d6ba7781e1009c660c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/ff485fafbff646d6ba7781e1009c660c 2024-11-15T14:16:27,619 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#A#compaction#363 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:27,619 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/e777bd0f049747fc8be8ed170d29c007 is 175, key is test_row_0/A:col10/1731680184821/Put/seqid=0 2024-11-15T14:16:27,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742258_1434 (size=32005) 2024-11-15T14:16:27,623 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/B of a6c59e08ce89fd58e535f1c83edb91e8 into ff485fafbff646d6ba7781e1009c660c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:27,623 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:27,623 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/B, priority=13, startTime=1731680187190; duration=0sec 2024-11-15T14:16:27,623 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:27,623 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:B 2024-11-15T14:16:27,623 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:27,624 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:27,624 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): a6c59e08ce89fd58e535f1c83edb91e8/C is initiating minor compaction (all files) 2024-11-15T14:16:27,624 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6c59e08ce89fd58e535f1c83edb91e8/C in TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:27,624 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/e0d5ff24efbf4681a3475b3e75cc0bfd, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/c18720776e584d3cbe3c97c22abea83c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/d6e7b24af5b14e46a753c7afdc0de5f0] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp, totalSize=36.6 K 2024-11-15T14:16:27,625 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting e0d5ff24efbf4681a3475b3e75cc0bfd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731680182654 2024-11-15T14:16:27,625 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c18720776e584d3cbe3c97c22abea83c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731680183521 2024-11-15T14:16:27,625 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting d6e7b24af5b14e46a753c7afdc0de5f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731680184817 2024-11-15T14:16:27,631 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6c59e08ce89fd58e535f1c83edb91e8#C#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:27,632 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/6c155baf518b484d96475e0987eb5880 is 50, key is test_row_0/C:col10/1731680184821/Put/seqid=0 2024-11-15T14:16:27,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742259_1435 (size=13051) 2024-11-15T14:16:27,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:27,727 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158ccfedb58ec841b6b3579b277aaab465_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158ccfedb58ec841b6b3579b277aaab465_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:27,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/e2f781e5ebcb4656bd5ca90905640132, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:27,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/e2f781e5ebcb4656bd5ca90905640132 is 175, key is test_row_0/A:col10/1731680185645/Put/seqid=0 2024-11-15T14:16:27,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742260_1436 (size=31255) 2024-11-15T14:16:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:27,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. as already flushing 2024-11-15T14:16:27,863 INFO [Thread-1567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1567. 2024-11-15T14:16:27,863 DEBUG [Thread-1567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:27,863 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,864 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:27,867 INFO [Thread-1569 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1569. 2024-11-15T14:16:27,867 DEBUG [Thread-1569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:27,867 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,867 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,867 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:27,868 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-15T14:16:27,868 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:27,868 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,868 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:27,869 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-11-15T14:16:27,869 INFO [Thread-1563 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1563. 2024-11-15T14:16:27,869 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:27,869 DEBUG [Thread-1563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:27,869 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,869 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,870 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,870 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:27,871 INFO [Thread-1558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1558. 2024-11-15T14:16:27,871 DEBUG [Thread-1558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:27,871 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,871 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:27,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:27,872 INFO [Thread-1554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1554. 2024-11-15T14:16:27,872 DEBUG [Thread-1554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:27,872 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,872 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,872 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:27,873 INFO [Thread-1552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1552. 2024-11-15T14:16:27,873 DEBUG [Thread-1552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:27,873 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,873 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,873 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:27,876 INFO [Thread-1571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1571. 2024-11-15T14:16:27,876 DEBUG [Thread-1571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:27,876 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,876 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:27,879 INFO [Thread-1565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1565. 2024-11-15T14:16:27,879 DEBUG [Thread-1565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:27,879 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,879 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:27,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T14:16:28,031 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/e777bd0f049747fc8be8ed170d29c007 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/e777bd0f049747fc8be8ed170d29c007 2024-11-15T14:16:28,039 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/A of a6c59e08ce89fd58e535f1c83edb91e8 into e777bd0f049747fc8be8ed170d29c007(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:28,039 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:28,039 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/A, priority=13, startTime=1731680187190; duration=0sec 2024-11-15T14:16:28,039 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:28,039 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:A 2024-11-15T14:16:28,043 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/6c155baf518b484d96475e0987eb5880 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/6c155baf518b484d96475e0987eb5880 2024-11-15T14:16:28,050 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6c59e08ce89fd58e535f1c83edb91e8/C of a6c59e08ce89fd58e535f1c83edb91e8 into 6c155baf518b484d96475e0987eb5880(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:28,050 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:28,050 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., storeName=a6c59e08ce89fd58e535f1c83edb91e8/C, priority=13, startTime=1731680187190; duration=0sec 2024-11-15T14:16:28,050 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:28,050 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6c59e08ce89fd58e535f1c83edb91e8:C 2024-11-15T14:16:28,132 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/e2f781e5ebcb4656bd5ca90905640132 2024-11-15T14:16:28,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/eea719c3237541b0b07d2a099b74090b is 50, key is test_row_0/B:col10/1731680185645/Put/seqid=0 2024-11-15T14:16:28,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742261_1437 (size=12301) 2024-11-15T14:16:28,588 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/eea719c3237541b0b07d2a099b74090b 2024-11-15T14:16:28,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/8d43a818042d400e8c873ed600a066c8 is 50, key is test_row_0/C:col10/1731680185645/Put/seqid=0 2024-11-15T14:16:28,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742262_1438 (size=12301) 2024-11-15T14:16:29,005 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/8d43a818042d400e8c873ed600a066c8 2024-11-15T14:16:29,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/e2f781e5ebcb4656bd5ca90905640132 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/e2f781e5ebcb4656bd5ca90905640132 2024-11-15T14:16:29,017 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/e2f781e5ebcb4656bd5ca90905640132, entries=150, sequenceid=352, filesize=30.5 K 2024-11-15T14:16:29,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/eea719c3237541b0b07d2a099b74090b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/eea719c3237541b0b07d2a099b74090b 2024-11-15T14:16:29,022 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/eea719c3237541b0b07d2a099b74090b, entries=150, sequenceid=352, filesize=12.0 K 2024-11-15T14:16:29,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/8d43a818042d400e8c873ed600a066c8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/8d43a818042d400e8c873ed600a066c8 2024-11-15T14:16:29,026 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/8d43a818042d400e8c873ed600a066c8, entries=150, sequenceid=352, filesize=12.0 K 2024-11-15T14:16:29,027 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for a6c59e08ce89fd58e535f1c83edb91e8 in 1716ms, sequenceid=352, compaction requested=false 2024-11-15T14:16:29,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for a6c59e08ce89fd58e535f1c83edb91e8: 2024-11-15T14:16:29,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:29,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-15T14:16:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-15T14:16:29,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-15T14:16:29,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2510 sec 2024-11-15T14:16:29,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 3.2570 sec 2024-11-15T14:16:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T14:16:29,942 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 649 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1947 rows 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 646 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1938 rows 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 649 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1947 rows 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 650 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1950 rows 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 655 2024-11-15T14:16:29,943 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1964 rows 2024-11-15T14:16:29,943 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T14:16:29,943 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T14:16:29,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:29,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:29,944 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:29,945 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T14:16:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T14:16:29,950 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680189950"}]},"ts":"1731680189950"} 2024-11-15T14:16:29,952 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T14:16:29,952 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T14:16:29,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T14:16:29,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, UNASSIGN}] 2024-11-15T14:16:29,955 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, UNASSIGN 2024-11-15T14:16:29,955 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=a6c59e08ce89fd58e535f1c83edb91e8, regionState=CLOSING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:16:29,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, UNASSIGN because future has completed 2024-11-15T14:16:29,957 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T14:16:29,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:16:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T14:16:30,109 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:30,109 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T14:16:30,110 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing a6c59e08ce89fd58e535f1c83edb91e8, disabling compactions & flushes 2024-11-15T14:16:30,110 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:30,110 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:30,110 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. after waiting 0 ms 2024-11-15T14:16:30,110 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:30,110 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing a6c59e08ce89fd58e535f1c83edb91e8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T14:16:30,110 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=A 2024-11-15T14:16:30,110 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:30,110 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=B 2024-11-15T14:16:30,110 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:30,110 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6c59e08ce89fd58e535f1c83edb91e8, store=C 2024-11-15T14:16:30,110 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:30,116 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f49146e4728e4035854f5cddf65b201a_a6c59e08ce89fd58e535f1c83edb91e8 is 50, key is test_row_0/A:col10/1731680187871/Put/seqid=0 2024-11-15T14:16:30,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742263_1439 (size=12454) 2024-11-15T14:16:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T14:16:30,519 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:30,522 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f49146e4728e4035854f5cddf65b201a_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f49146e4728e4035854f5cddf65b201a_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:30,523 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/3193d48a7603416a9878537e3ca5042a, store: [table=TestAcidGuarantees family=A region=a6c59e08ce89fd58e535f1c83edb91e8] 2024-11-15T14:16:30,524 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/3193d48a7603416a9878537e3ca5042a is 175, key is test_row_0/A:col10/1731680187871/Put/seqid=0 2024-11-15T14:16:30,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742264_1440 (size=31255) 2024-11-15T14:16:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T14:16:30,928 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/3193d48a7603416a9878537e3ca5042a 2024-11-15T14:16:30,938 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/e72ff7101fc3494da24b473a76cab881 is 50, key is test_row_0/B:col10/1731680187871/Put/seqid=0 2024-11-15T14:16:30,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742265_1441 (size=12301) 2024-11-15T14:16:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T14:16:31,343 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/e72ff7101fc3494da24b473a76cab881 2024-11-15T14:16:31,355 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/d17b700764fc45839b6683556bbfe2bf is 50, key is test_row_0/C:col10/1731680187871/Put/seqid=0 2024-11-15T14:16:31,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742266_1442 (size=12301) 2024-11-15T14:16:31,760 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/d17b700764fc45839b6683556bbfe2bf 2024-11-15T14:16:31,766 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/A/3193d48a7603416a9878537e3ca5042a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/3193d48a7603416a9878537e3ca5042a 2024-11-15T14:16:31,771 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/3193d48a7603416a9878537e3ca5042a, entries=150, sequenceid=365, filesize=30.5 K 2024-11-15T14:16:31,772 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/B/e72ff7101fc3494da24b473a76cab881 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/e72ff7101fc3494da24b473a76cab881 2024-11-15T14:16:31,776 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/e72ff7101fc3494da24b473a76cab881, entries=150, sequenceid=365, filesize=12.0 K 2024-11-15T14:16:31,777 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/.tmp/C/d17b700764fc45839b6683556bbfe2bf as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/d17b700764fc45839b6683556bbfe2bf 2024-11-15T14:16:31,780 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/d17b700764fc45839b6683556bbfe2bf, entries=150, sequenceid=365, filesize=12.0 K 2024-11-15T14:16:31,781 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for a6c59e08ce89fd58e535f1c83edb91e8 in 1670ms, sequenceid=365, compaction requested=true 2024-11-15T14:16:31,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/03c064cd6d594b9aac1e0f8370cf6e28, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/4e8bc4fb4a164bf99fdd8577a7fb131f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/589aa28041ba49079a8e9dbe768ae2da, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/ba33dc3c4fd347d98cd7350b0c6b6f54, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/459b7b7a1dc5453ea6b1c0be5a32cb3c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/6481f08b23dc4a9a96fdaba32e418644, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/45aea5716f8941af8ff7f9d6a01aa5ff, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/68bf2268e2e14f4eae940e94a6a836a7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/907563f9567b49779ef8cb6ccd5a5633, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/fd30a11e7a44420783557dac5a6fe92b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/c23a4673412d475c80583916723e1015, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/35e8d143e6c547c180125a63bb8c7d07, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f86a86924830480a8bba8a1809490fd4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/b6d04846e64d4f08889559fd86ee8e08, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/3fb0bb5b14134db28674743355af5fcb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5951ea7fe2f14d9391cd69b33b2314af, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/622f7bfe8ca64534bdfd751bc1dceac3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f3f8931124774db29baf665c3f825599, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/90f7dbf143554af9b882df75f9da1ee2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/fdeda0a0176849df82e38c9aaba48e51, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/105020222b814b248296e656d4dceff7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5ff830a6f7b64040976bc11be5789178] to archive 2024-11-15T14:16:31,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:16:31,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/03c064cd6d594b9aac1e0f8370cf6e28 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/03c064cd6d594b9aac1e0f8370cf6e28 2024-11-15T14:16:31,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/4e8bc4fb4a164bf99fdd8577a7fb131f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/4e8bc4fb4a164bf99fdd8577a7fb131f 2024-11-15T14:16:31,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/589aa28041ba49079a8e9dbe768ae2da to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/589aa28041ba49079a8e9dbe768ae2da 2024-11-15T14:16:31,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/ba33dc3c4fd347d98cd7350b0c6b6f54 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/ba33dc3c4fd347d98cd7350b0c6b6f54 2024-11-15T14:16:31,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/459b7b7a1dc5453ea6b1c0be5a32cb3c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/459b7b7a1dc5453ea6b1c0be5a32cb3c 2024-11-15T14:16:31,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/6481f08b23dc4a9a96fdaba32e418644 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/6481f08b23dc4a9a96fdaba32e418644 2024-11-15T14:16:31,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/45aea5716f8941af8ff7f9d6a01aa5ff to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/45aea5716f8941af8ff7f9d6a01aa5ff 2024-11-15T14:16:31,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/68bf2268e2e14f4eae940e94a6a836a7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/68bf2268e2e14f4eae940e94a6a836a7 2024-11-15T14:16:31,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/907563f9567b49779ef8cb6ccd5a5633 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/907563f9567b49779ef8cb6ccd5a5633 2024-11-15T14:16:31,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/fd30a11e7a44420783557dac5a6fe92b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/fd30a11e7a44420783557dac5a6fe92b 2024-11-15T14:16:31,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/c23a4673412d475c80583916723e1015 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/c23a4673412d475c80583916723e1015 2024-11-15T14:16:31,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/35e8d143e6c547c180125a63bb8c7d07 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/35e8d143e6c547c180125a63bb8c7d07 2024-11-15T14:16:31,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f86a86924830480a8bba8a1809490fd4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f86a86924830480a8bba8a1809490fd4 2024-11-15T14:16:31,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/b6d04846e64d4f08889559fd86ee8e08 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/b6d04846e64d4f08889559fd86ee8e08 2024-11-15T14:16:31,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/3fb0bb5b14134db28674743355af5fcb to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/3fb0bb5b14134db28674743355af5fcb 2024-11-15T14:16:31,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5951ea7fe2f14d9391cd69b33b2314af to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5951ea7fe2f14d9391cd69b33b2314af 2024-11-15T14:16:31,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/622f7bfe8ca64534bdfd751bc1dceac3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/622f7bfe8ca64534bdfd751bc1dceac3 2024-11-15T14:16:31,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f3f8931124774db29baf665c3f825599 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/f3f8931124774db29baf665c3f825599 2024-11-15T14:16:31,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/90f7dbf143554af9b882df75f9da1ee2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/90f7dbf143554af9b882df75f9da1ee2 2024-11-15T14:16:31,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/fdeda0a0176849df82e38c9aaba48e51 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/fdeda0a0176849df82e38c9aaba48e51 2024-11-15T14:16:31,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/105020222b814b248296e656d4dceff7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/105020222b814b248296e656d4dceff7 2024-11-15T14:16:31,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5ff830a6f7b64040976bc11be5789178 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/5ff830a6f7b64040976bc11be5789178 2024-11-15T14:16:31,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/0bf6989de027496485ef5f1b1101ee13, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/a7b631b8fc5d49efa3b48f5600ed323f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/1398ddab04ba40b8ac3e48ceaf6bf1d3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/6da36d8b5a44415f8ddefc542f68588d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/9289591552194a069447cf1024618786, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c1ee17400dfb4ddd9ae1470a9aefac90, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/acdafd629f2d44df851d2a507b08f4e2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c9ac51259ac04544b02f17275c53b48d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/eb2f366e9bc945e49acf4d5c906e760a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5f1c98de8b564bce858be674dad8be40, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5b87236d4d1144a28d3bf5a57003fc2d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/54853d1f880b4d80b69e5e1ca8ee4269, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5a68e1f2d4bc4ab99db5faa387408297, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/333bed2f402c4b628757347c5be0589e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c3426dde38334d59b2242c06b2b295ea, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/a169a5788c6e4ea995077a3be47ab5b3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/11445411584c4988806755c4651ff89d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/dcb297757347487b83f87adcbf0afedf, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/8f61716f0c51464195bf645769337f27, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/3c56a486cc79483da107ec2ff2ad5435, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/b6bd24e38fad4abf91e6744f56c3425b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/43c437bb69524c0eae83a3cf33e297bd] to archive 2024-11-15T14:16:31,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:16:31,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/0bf6989de027496485ef5f1b1101ee13 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/0bf6989de027496485ef5f1b1101ee13 2024-11-15T14:16:31,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/a7b631b8fc5d49efa3b48f5600ed323f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/a7b631b8fc5d49efa3b48f5600ed323f 2024-11-15T14:16:31,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/1398ddab04ba40b8ac3e48ceaf6bf1d3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/1398ddab04ba40b8ac3e48ceaf6bf1d3 2024-11-15T14:16:31,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/6da36d8b5a44415f8ddefc542f68588d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/6da36d8b5a44415f8ddefc542f68588d 2024-11-15T14:16:31,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/9289591552194a069447cf1024618786 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/9289591552194a069447cf1024618786 2024-11-15T14:16:31,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c1ee17400dfb4ddd9ae1470a9aefac90 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c1ee17400dfb4ddd9ae1470a9aefac90 2024-11-15T14:16:31,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/acdafd629f2d44df851d2a507b08f4e2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/acdafd629f2d44df851d2a507b08f4e2 2024-11-15T14:16:31,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c9ac51259ac04544b02f17275c53b48d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c9ac51259ac04544b02f17275c53b48d 2024-11-15T14:16:31,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/eb2f366e9bc945e49acf4d5c906e760a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/eb2f366e9bc945e49acf4d5c906e760a 2024-11-15T14:16:31,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5f1c98de8b564bce858be674dad8be40 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5f1c98de8b564bce858be674dad8be40 2024-11-15T14:16:31,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5b87236d4d1144a28d3bf5a57003fc2d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5b87236d4d1144a28d3bf5a57003fc2d 2024-11-15T14:16:31,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/54853d1f880b4d80b69e5e1ca8ee4269 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/54853d1f880b4d80b69e5e1ca8ee4269 2024-11-15T14:16:31,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5a68e1f2d4bc4ab99db5faa387408297 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/5a68e1f2d4bc4ab99db5faa387408297 2024-11-15T14:16:31,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/333bed2f402c4b628757347c5be0589e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/333bed2f402c4b628757347c5be0589e 2024-11-15T14:16:31,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c3426dde38334d59b2242c06b2b295ea to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/c3426dde38334d59b2242c06b2b295ea 2024-11-15T14:16:31,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/a169a5788c6e4ea995077a3be47ab5b3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/a169a5788c6e4ea995077a3be47ab5b3 2024-11-15T14:16:31,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/11445411584c4988806755c4651ff89d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/11445411584c4988806755c4651ff89d 2024-11-15T14:16:31,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/dcb297757347487b83f87adcbf0afedf to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/dcb297757347487b83f87adcbf0afedf 2024-11-15T14:16:31,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/8f61716f0c51464195bf645769337f27 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/8f61716f0c51464195bf645769337f27 2024-11-15T14:16:31,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/3c56a486cc79483da107ec2ff2ad5435 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/3c56a486cc79483da107ec2ff2ad5435 2024-11-15T14:16:31,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/b6bd24e38fad4abf91e6744f56c3425b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/b6bd24e38fad4abf91e6744f56c3425b 2024-11-15T14:16:31,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/43c437bb69524c0eae83a3cf33e297bd to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/43c437bb69524c0eae83a3cf33e297bd 2024-11-15T14:16:31,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/520d7a608e5a4c269ad89eefa7b1f718, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/efa8b675c25a44448b64bbbe97c2aec1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/2b3872e5a97b4f4e828d748430388d63, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/7c697ce2211946a882740baf3b1dc4ea, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/76091b99cc2c4714b4c1ed25cd348352, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/b4214c8341d747efbd17c8d526d5e326, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/02377d5dc18546038192b4f5f7c4249a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/29a84ad777c243cca92f516478280c80, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/77fb4d4ef4be4fb0befffdb7750ecef6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/5ed1e6288bc04b65a5a2f8207c839129, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/08b1cb41325c4197835b4eb6e0589b98, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/2665cd52a25d4b508d72e075cecc8366, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/9444b4bf7a84456ea6d063cddc26b445, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/794a704b499f4cafb91e0cc324f66e7b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/b098f53f31114293b23b9e24a568bcc1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/c5c36c9b93ba4ba9aeaa23826fa9797f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/a93271113b474450bf177128ef1c7288, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/49edd05e2b7d48a39c48f4b08b525506, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/e0d5ff24efbf4681a3475b3e75cc0bfd, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/dc393122b0584055b78fcc2ec9db6308, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/c18720776e584d3cbe3c97c22abea83c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/d6e7b24af5b14e46a753c7afdc0de5f0] to archive 2024-11-15T14:16:31,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:16:31,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/520d7a608e5a4c269ad89eefa7b1f718 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/520d7a608e5a4c269ad89eefa7b1f718 2024-11-15T14:16:31,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/efa8b675c25a44448b64bbbe97c2aec1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/efa8b675c25a44448b64bbbe97c2aec1 2024-11-15T14:16:31,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/2b3872e5a97b4f4e828d748430388d63 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/2b3872e5a97b4f4e828d748430388d63 2024-11-15T14:16:31,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/7c697ce2211946a882740baf3b1dc4ea to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/7c697ce2211946a882740baf3b1dc4ea 2024-11-15T14:16:31,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/76091b99cc2c4714b4c1ed25cd348352 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/76091b99cc2c4714b4c1ed25cd348352 2024-11-15T14:16:31,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/b4214c8341d747efbd17c8d526d5e326 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/b4214c8341d747efbd17c8d526d5e326 2024-11-15T14:16:31,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/02377d5dc18546038192b4f5f7c4249a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/02377d5dc18546038192b4f5f7c4249a 2024-11-15T14:16:31,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/29a84ad777c243cca92f516478280c80 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/29a84ad777c243cca92f516478280c80 2024-11-15T14:16:31,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/77fb4d4ef4be4fb0befffdb7750ecef6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/77fb4d4ef4be4fb0befffdb7750ecef6 2024-11-15T14:16:31,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/5ed1e6288bc04b65a5a2f8207c839129 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/5ed1e6288bc04b65a5a2f8207c839129 2024-11-15T14:16:31,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/08b1cb41325c4197835b4eb6e0589b98 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/08b1cb41325c4197835b4eb6e0589b98 2024-11-15T14:16:31,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/2665cd52a25d4b508d72e075cecc8366 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/2665cd52a25d4b508d72e075cecc8366 2024-11-15T14:16:31,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/9444b4bf7a84456ea6d063cddc26b445 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/9444b4bf7a84456ea6d063cddc26b445 2024-11-15T14:16:31,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/794a704b499f4cafb91e0cc324f66e7b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/794a704b499f4cafb91e0cc324f66e7b 2024-11-15T14:16:31,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/b098f53f31114293b23b9e24a568bcc1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/b098f53f31114293b23b9e24a568bcc1 2024-11-15T14:16:31,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/c5c36c9b93ba4ba9aeaa23826fa9797f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/c5c36c9b93ba4ba9aeaa23826fa9797f 2024-11-15T14:16:31,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/a93271113b474450bf177128ef1c7288 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/a93271113b474450bf177128ef1c7288 2024-11-15T14:16:31,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/49edd05e2b7d48a39c48f4b08b525506 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/49edd05e2b7d48a39c48f4b08b525506 2024-11-15T14:16:31,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/e0d5ff24efbf4681a3475b3e75cc0bfd to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/e0d5ff24efbf4681a3475b3e75cc0bfd 2024-11-15T14:16:31,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/dc393122b0584055b78fcc2ec9db6308 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/dc393122b0584055b78fcc2ec9db6308 2024-11-15T14:16:31,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/c18720776e584d3cbe3c97c22abea83c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/c18720776e584d3cbe3c97c22abea83c 2024-11-15T14:16:31,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/d6e7b24af5b14e46a753c7afdc0de5f0 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/d6e7b24af5b14e46a753c7afdc0de5f0 2024-11-15T14:16:31,851 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/recovered.edits/368.seqid, newMaxSeqId=368, maxSeqId=4 2024-11-15T14:16:31,852 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. 2024-11-15T14:16:31,852 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for a6c59e08ce89fd58e535f1c83edb91e8: Waiting for close lock at 1731680190110Running coprocessor pre-close hooks at 1731680190110Disabling compacts and flushes for region at 1731680190110Disabling writes for close at 1731680190110Obtaining lock to block concurrent updates at 1731680190110Preparing flush snapshotting stores in a6c59e08ce89fd58e535f1c83edb91e8 at 1731680190110Finished memstore snapshotting TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8., syncing WAL and waiting on mvcc, flushsize=dataSize=48090, getHeapSize=143520, getOffHeapSize=0, getCellsCount=1050 at 1731680190110Flushing stores of TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8. at 1731680190111 (+1 ms)Flushing a6c59e08ce89fd58e535f1c83edb91e8/A: creating writer at 1731680190111Flushing mob file a6c59e08ce89fd58e535f1c83edb91e8/A: appending metadata at 1731680190116 (+5 ms)Flushing mob file a6c59e08ce89fd58e535f1c83edb91e8/A: closing flushed file at 1731680190116Flushing a6c59e08ce89fd58e535f1c83edb91e8/A: appending metadata at 1731680190523 (+407 ms)Flushing a6c59e08ce89fd58e535f1c83edb91e8/A: closing flushed file at 1731680190523Flushing a6c59e08ce89fd58e535f1c83edb91e8/B: creating writer at 1731680190934 (+411 ms)Flushing a6c59e08ce89fd58e535f1c83edb91e8/B: appending metadata at 1731680190938 (+4 ms)Flushing a6c59e08ce89fd58e535f1c83edb91e8/B: closing flushed file at 1731680190938Flushing a6c59e08ce89fd58e535f1c83edb91e8/C: creating writer at 1731680191350 (+412 ms)Flushing a6c59e08ce89fd58e535f1c83edb91e8/C: appending metadata at 1731680191354 (+4 ms)Flushing a6c59e08ce89fd58e535f1c83edb91e8/C: closing flushed file at 1731680191354Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@175d2d35: reopening flushed file at 1731680191766 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c83f7d9: reopening flushed file at 1731680191771 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@417cf5ba: reopening flushed file at 1731680191776 (+5 ms)Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for a6c59e08ce89fd58e535f1c83edb91e8 in 1670ms, sequenceid=365, compaction requested=true at 1731680191781 (+5 ms)Writing region close event to WAL at 1731680191847 (+66 ms)Running coprocessor post-close hooks at 1731680191852 (+5 ms)Closed at 1731680191852 2024-11-15T14:16:31,854 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:31,854 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=a6c59e08ce89fd58e535f1c83edb91e8, regionState=CLOSED 2024-11-15T14:16:31,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:16:31,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-15T14:16:31,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure a6c59e08ce89fd58e535f1c83edb91e8, server=26754e923c4a,40177,1731680077530 in 1.9000 sec 2024-11-15T14:16:31,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-15T14:16:31,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6c59e08ce89fd58e535f1c83edb91e8, UNASSIGN in 1.9060 sec 2024-11-15T14:16:31,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-15T14:16:31,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9090 sec 2024-11-15T14:16:31,866 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680191865"}]},"ts":"1731680191865"} 2024-11-15T14:16:31,867 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T14:16:31,867 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T14:16:31,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9230 sec 2024-11-15T14:16:32,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T14:16:32,093 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:32,093 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T14:16:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:32,095 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T14:16:32,096 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:32,100 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,102 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/recovered.edits] 2024-11-15T14:16:32,105 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/3193d48a7603416a9878537e3ca5042a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/3193d48a7603416a9878537e3ca5042a 2024-11-15T14:16:32,105 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/e2f781e5ebcb4656bd5ca90905640132 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/e2f781e5ebcb4656bd5ca90905640132 2024-11-15T14:16:32,106 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/e777bd0f049747fc8be8ed170d29c007 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/A/e777bd0f049747fc8be8ed170d29c007 2024-11-15T14:16:32,110 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/e72ff7101fc3494da24b473a76cab881 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/e72ff7101fc3494da24b473a76cab881 2024-11-15T14:16:32,111 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/eea719c3237541b0b07d2a099b74090b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/eea719c3237541b0b07d2a099b74090b 2024-11-15T14:16:32,112 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/ff485fafbff646d6ba7781e1009c660c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/B/ff485fafbff646d6ba7781e1009c660c 2024-11-15T14:16:32,114 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/6c155baf518b484d96475e0987eb5880 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/6c155baf518b484d96475e0987eb5880 2024-11-15T14:16:32,117 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/8d43a818042d400e8c873ed600a066c8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/8d43a818042d400e8c873ed600a066c8 2024-11-15T14:16:32,118 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/d17b700764fc45839b6683556bbfe2bf to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/C/d17b700764fc45839b6683556bbfe2bf 2024-11-15T14:16:32,120 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/recovered.edits/368.seqid to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8/recovered.edits/368.seqid 2024-11-15T14:16:32,120 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,121 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T14:16:32,121 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T14:16:32,122 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T14:16:32,125 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151f3fd66dec634de0a2aee0d2ed2f3b9e_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151f3fd66dec634de0a2aee0d2ed2f3b9e_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,126 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152e77a54520ca43449eba0a159eda33f8_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152e77a54520ca43449eba0a159eda33f8_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,127 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153c29796597c746368a837cccee73a152_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153c29796597c746368a837cccee73a152_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,128 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155c3aa1b29c844212aae8a8389aea4fad_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155c3aa1b29c844212aae8a8389aea4fad_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,129 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155f845048b99f44bb85cef8f7a92b97c3_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155f845048b99f44bb85cef8f7a92b97c3_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,130 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115636658d9c43b447b83e9162f877c5096_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115636658d9c43b447b83e9162f877c5096_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,131 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111569c8b891e0714a89ba4f8502c8a5c151_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111569c8b891e0714a89ba4f8502c8a5c151_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,132 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c75fdef670045a892c6ce3d433c6136_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c75fdef670045a892c6ce3d433c6136_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,133 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158ccfedb58ec841b6b3579b277aaab465_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158ccfedb58ec841b6b3579b277aaab465_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,134 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159a17cd75f4574b4aae809aba6b53d340_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159a17cd75f4574b4aae809aba6b53d340_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,135 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a4fb4ddeab124921a971c5754870e2b5_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a4fb4ddeab124921a971c5754870e2b5_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,136 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a6e0ee26198f41738d896de8f8016d2f_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a6e0ee26198f41738d896de8f8016d2f_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,137 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a84d7b7f7ab8426797176346625c2c2e_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a84d7b7f7ab8426797176346625c2c2e_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,138 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b09412a803c14b8e8f53b151fb61f2c0_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b09412a803c14b8e8f53b151fb61f2c0_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,139 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e116bda9c2e1422e84dccda98a33d07b_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e116bda9c2e1422e84dccda98a33d07b_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,140 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f49146e4728e4035854f5cddf65b201a_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f49146e4728e4035854f5cddf65b201a_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,141 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f4d01da7b0d34de7875be221f06f4fc1_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f4d01da7b0d34de7875be221f06f4fc1_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,142 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f78ef9524e2a4403956eae86058dd39e_a6c59e08ce89fd58e535f1c83edb91e8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f78ef9524e2a4403956eae86058dd39e_a6c59e08ce89fd58e535f1c83edb91e8 2024-11-15T14:16:32,142 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T14:16:32,146 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:32,148 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T14:16:32,150 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T14:16:32,151 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:32,151 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T14:16:32,152 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731680192151"}]},"ts":"9223372036854775807"} 2024-11-15T14:16:32,154 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T14:16:32,154 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a6c59e08ce89fd58e535f1c83edb91e8, NAME => 'TestAcidGuarantees,,1731680164592.a6c59e08ce89fd58e535f1c83edb91e8.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T14:16:32,154 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T14:16:32,154 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731680192154"}]},"ts":"9223372036854775807"} 2024-11-15T14:16:32,156 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T14:16:32,157 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:32,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-11-15T14:16:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T14:16:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T14:16:32,205 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:32,211 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1762377796_22 at /127.0.0.1:52412 [Waiting for operation #812] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1762377796_22 at /127.0.0.1:37092 [Waiting for operation #779] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1873743887_22 at /127.0.0.1:53186 [Waiting for operation #199] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1873743887_22 at /127.0.0.1:53162 [Waiting for operation #676] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=802 (was 771) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=714 (was 260) - AvailableMemoryMB LEAK? - 2024-11-15T14:16:32,221 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=802, ProcessCount=11, AvailableMemoryMB=713 2024-11-15T14:16:32,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T14:16:32,223 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T14:16:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:32,225 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T14:16:32,225 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:32,225 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-15T14:16:32,226 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T14:16:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-15T14:16:32,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742267_1443 (size=1061) 2024-11-15T14:16:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-15T14:16:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-15T14:16:32,633 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931 2024-11-15T14:16:32,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742268_1444 (size=53) 2024-11-15T14:16:32,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-15T14:16:33,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:16:33,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f24e805c31c506536abbd73db35b0c0f, disabling compactions & flushes 2024-11-15T14:16:33,038 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:33,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:33,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. after waiting 0 ms 2024-11-15T14:16:33,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:33,038 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:33,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f24e805c31c506536abbd73db35b0c0f: Waiting for close lock at 1731680193038Disabling compacts and flushes for region at 1731680193038Disabling writes for close at 1731680193038Writing region close event to WAL at 1731680193038Closed at 1731680193038 2024-11-15T14:16:33,039 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T14:16:33,039 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731680193039"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731680193039"}]},"ts":"1731680193039"} 2024-11-15T14:16:33,040 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T14:16:33,041 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T14:16:33,041 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680193041"}]},"ts":"1731680193041"} 2024-11-15T14:16:33,045 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T14:16:33,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f24e805c31c506536abbd73db35b0c0f, ASSIGN}] 2024-11-15T14:16:33,048 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f24e805c31c506536abbd73db35b0c0f, ASSIGN 2024-11-15T14:16:33,049 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f24e805c31c506536abbd73db35b0c0f, ASSIGN; state=OFFLINE, location=26754e923c4a,40177,1731680077530; forceNewPlan=false, retain=false 2024-11-15T14:16:33,199 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=f24e805c31c506536abbd73db35b0c0f, regionState=OPENING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:16:33,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f24e805c31c506536abbd73db35b0c0f, ASSIGN because future has completed 2024-11-15T14:16:33,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:16:33,356 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:33,356 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} 2024-11-15T14:16:33,357 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:33,357 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:16:33,357 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:33,357 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:33,358 INFO [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:33,359 INFO [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:16:33,360 INFO [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f24e805c31c506536abbd73db35b0c0f columnFamilyName A 2024-11-15T14:16:33,360 DEBUG [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:33,360 INFO [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] regionserver.HStore(327): Store=f24e805c31c506536abbd73db35b0c0f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:16:33,360 INFO [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:33,361 INFO [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:16:33,361 INFO [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f24e805c31c506536abbd73db35b0c0f columnFamilyName B 2024-11-15T14:16:33,361 DEBUG [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-15T14:16:33,363 INFO [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] regionserver.HStore(327): Store=f24e805c31c506536abbd73db35b0c0f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:16:33,363 INFO [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:33,364 INFO [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:16:33,365 INFO [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f24e805c31c506536abbd73db35b0c0f columnFamilyName C 2024-11-15T14:16:33,365 DEBUG [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:16:33,367 INFO [StoreOpener-f24e805c31c506536abbd73db35b0c0f-1 {}] regionserver.HStore(327): Store=f24e805c31c506536abbd73db35b0c0f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:16:33,367 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:33,367 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:33,369 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:33,369 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:33,370 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:33,370 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:33,372 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T14:16:33,374 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:33,376 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T14:16:33,377 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened f24e805c31c506536abbd73db35b0c0f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63729323, jitterRate=-0.0503590852022171}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T14:16:33,377 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:33,378 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for f24e805c31c506536abbd73db35b0c0f: Running coprocessor pre-open hook at 1731680193357Writing region info on filesystem at 1731680193357Initializing all the Stores at 1731680193358 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680193358Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680193358Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680193358Cleaning up temporary data from old regions at 1731680193370 (+12 ms)Running coprocessor post-open hooks at 1731680193377 (+7 ms)Region opened successfully at 1731680193378 (+1 ms) 2024-11-15T14:16:33,379 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., pid=122, masterSystemTime=1731680193353 2024-11-15T14:16:33,381 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:33,381 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=f24e805c31c506536abbd73db35b0c0f, regionState=OPEN, openSeqNum=2, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:16:33,383 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:33,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:16:33,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-15T14:16:33,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 in 184 msec 2024-11-15T14:16:33,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-15T14:16:33,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f24e805c31c506536abbd73db35b0c0f, ASSIGN in 341 msec 2024-11-15T14:16:33,389 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T14:16:33,390 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680193389"}]},"ts":"1731680193389"} 2024-11-15T14:16:33,391 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T14:16:33,392 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T14:16:33,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-11-15T14:16:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-15T14:16:34,372 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:34,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,373 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:34,373 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:34,374 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4927bcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:34,376 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:34,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc574c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,378 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:34,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:34,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:34,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:34,382 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T14:16:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 26754e923c4a,35807,1731680075796 2024-11-15T14:16:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@631b8882 2024-11-15T14:16:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T14:16:34,385 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T14:16:34,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:34,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:34,387 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d3bafc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:34,391 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:34,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@566dcab2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,403 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:34,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:34,404 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29baf5b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:34,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:34,406 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:34,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d4ba3b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:34,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:34,412 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72baccc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:34,414 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683e7153, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:34,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:34,417 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@356645a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:34,419 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@334b7dbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:34,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:34,421 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d73134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:34,422 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:34,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c17aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,423 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:34,423 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:34,424 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:34,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:34,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:34,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f9aafc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:34,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:34,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:34,426 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f3ba426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:34,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:34,432 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:34,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:34,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:34,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d3df677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:34,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:34,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:34,434 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a2fb832, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:34,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:34,444 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:34,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:34,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,446 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:34,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:34,447 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:34,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28b8bd22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:34,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:34,450 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:34,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:34,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f73fc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:34,452 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@569c1d98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:16:34,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:16:34,456 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:16:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:16:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:16:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cf9aa64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:16:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:16:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:34,458 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:16:34,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2303c25f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:16:34,463 DEBUG [Thread-2000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:34,463 DEBUG [Thread-2008 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:34,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:34,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:34,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:16:34,471 DEBUG [Thread-1991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:34,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:34,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:34,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:34,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:16:34,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:16:34,479 DEBUG [Thread-2006 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:34,480 DEBUG [Thread-1989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:34,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:34,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:34,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:16:34,487 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:16:34,489 DEBUG [Thread-2004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:34,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:34,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-15T14:16:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T14:16:34,493 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:34,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:34,494 DEBUG [Thread-1995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:16:34,495 DEBUG [Thread-1995 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:34,495 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:34,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:34,495 DEBUG [Thread-1995 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:34,497 DEBUG [Thread-1997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:34,498 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:34,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:34,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:34,503 DEBUG [Thread-2002 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:16:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:34,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:16:34,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:16:34,504 DEBUG [Thread-2002 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:16:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:16:34,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:34,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:16:34,510 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:16:34,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2] 2024-11-15T14:16:34,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:34,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T14:16:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:34,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/a279978ce3284751b5a7a86df67d24ce is 50, key is test_row_0/A:col10/1731680194495/Put/seqid=0 2024-11-15T14:16:34,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742269_1445 (size=16681) 2024-11-15T14:16:34,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731680194534, max timestamp=1731680194587Num uniques 859; 2024-11-15T14:16:34,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T14:16:34,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731680194534, max timestamp=1731680194587Num uniques 859; 2024-11-15T14:16:34,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T14:16:34,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731680194534, max timestamp=1731680194587Num uniques 859; 2024-11-15T14:16:34,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T14:16:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T14:16:34,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T14:16:34,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:34,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:34,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:34,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:34,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:34,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680254652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:34,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:34,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680254655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:34,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680254657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:34,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:34,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680254661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:34,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:34,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680254674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:34,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T14:16:34,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:34,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:34,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:34,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:34,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:34,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680254785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:34,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680254797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:34,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680254798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:34,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680254798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:34,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680254799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:34,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:34,812 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T14:16:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T14:16:34,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T14:16:34,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:34,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:34,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:34,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:34,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:34,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/a279978ce3284751b5a7a86df67d24ce 2024-11-15T14:16:35,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680255012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/562e143aa02f4750a96a57f5e478e7c6 is 50, key is test_row_0/B:col10/1731680194495/Put/seqid=0 2024-11-15T14:16:35,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680255012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680255013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680255013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680255022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742270_1446 (size=12001) 2024-11-15T14:16:35,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/562e143aa02f4750a96a57f5e478e7c6 2024-11-15T14:16:35,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T14:16:35,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:35,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:35,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:35,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:35,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T14:16:35,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/9d6497216bbf44d7b9fa5da2e73a56c4 is 50, key is test_row_0/C:col10/1731680194495/Put/seqid=0 2024-11-15T14:16:35,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742271_1447 (size=12001) 2024-11-15T14:16:35,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T14:16:35,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:35,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:35,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:35,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:35,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680255331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680255342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680255342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680255352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680255361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T14:16:35,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:35,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:35,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:35,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:35,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:35,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/9d6497216bbf44d7b9fa5da2e73a56c4 2024-11-15T14:16:35,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/a279978ce3284751b5a7a86df67d24ce as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a279978ce3284751b5a7a86df67d24ce 2024-11-15T14:16:35,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T14:16:35,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:35,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:35,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:35,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:35,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:35,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a279978ce3284751b5a7a86df67d24ce, entries=250, sequenceid=11, filesize=16.3 K 2024-11-15T14:16:35,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/562e143aa02f4750a96a57f5e478e7c6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/562e143aa02f4750a96a57f5e478e7c6 2024-11-15T14:16:35,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/562e143aa02f4750a96a57f5e478e7c6, entries=150, sequenceid=11, filesize=11.7 K 2024-11-15T14:16:35,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/9d6497216bbf44d7b9fa5da2e73a56c4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/9d6497216bbf44d7b9fa5da2e73a56c4 2024-11-15T14:16:35,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/9d6497216bbf44d7b9fa5da2e73a56c4, entries=150, sequenceid=11, filesize=11.7 K 2024-11-15T14:16:35,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f24e805c31c506536abbd73db35b0c0f in 1099ms, sequenceid=11, compaction requested=false 2024-11-15T14:16:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T14:16:35,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T14:16:35,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:35,735 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-15T14:16:35,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:35,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:35,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:35,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:35,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:35,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:35,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/6eabb2a05ef149aeac10b30d0566c72d is 50, key is test_row_0/A:col10/1731680194640/Put/seqid=0 2024-11-15T14:16:35,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742272_1448 (size=12001) 2024-11-15T14:16:35,802 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/6eabb2a05ef149aeac10b30d0566c72d 2024-11-15T14:16:35,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/34a57699e0d2456fae745b9aeb5971b3 is 50, key is test_row_0/B:col10/1731680194640/Put/seqid=0 2024-11-15T14:16:35,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:35,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742273_1449 (size=12001) 2024-11-15T14:16:35,865 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/34a57699e0d2456fae745b9aeb5971b3 2024-11-15T14:16:35,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/e1386746a6d04ee9a9462e8f1997684e is 50, key is test_row_0/C:col10/1731680194640/Put/seqid=0 2024-11-15T14:16:35,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680255883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680255884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680255886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680255901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680255904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:35,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:35,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:35,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742274_1450 (size=12001) 2024-11-15T14:16:35,931 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/e1386746a6d04ee9a9462e8f1997684e 2024-11-15T14:16:35,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/6eabb2a05ef149aeac10b30d0566c72d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/6eabb2a05ef149aeac10b30d0566c72d 2024-11-15T14:16:35,952 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/6eabb2a05ef149aeac10b30d0566c72d, entries=150, sequenceid=46, filesize=11.7 K 2024-11-15T14:16:35,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/34a57699e0d2456fae745b9aeb5971b3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/34a57699e0d2456fae745b9aeb5971b3 2024-11-15T14:16:35,966 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/34a57699e0d2456fae745b9aeb5971b3, entries=150, sequenceid=46, filesize=11.7 K 2024-11-15T14:16:35,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/e1386746a6d04ee9a9462e8f1997684e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e1386746a6d04ee9a9462e8f1997684e 2024-11-15T14:16:35,971 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e1386746a6d04ee9a9462e8f1997684e, entries=150, sequenceid=46, filesize=11.7 K 2024-11-15T14:16:35,972 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for f24e805c31c506536abbd73db35b0c0f in 238ms, sequenceid=46, compaction requested=false 2024-11-15T14:16:35,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:35,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:35,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-15T14:16:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-15T14:16:35,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-15T14:16:35,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-11-15T14:16:35,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4910 sec 2024-11-15T14:16:36,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T14:16:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:36,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:36,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:36,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:36,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:36,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:36,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:36,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/6ec9127757244dc097397549e3dd09d5 is 50, key is test_row_0/A:col10/1731680196018/Put/seqid=0 2024-11-15T14:16:36,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742275_1451 (size=14337) 2024-11-15T14:16:36,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/6ec9127757244dc097397549e3dd09d5 2024-11-15T14:16:36,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/79c02d9c4f5c44f8b38e6507e86b0aa8 is 50, key is test_row_0/B:col10/1731680196018/Put/seqid=0 2024-11-15T14:16:36,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680256076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680256081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680256083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680256089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742276_1452 (size=9657) 2024-11-15T14:16:36,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/79c02d9c4f5c44f8b38e6507e86b0aa8 2024-11-15T14:16:36,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/0e0082c7b7ca435380cad96a1dbff365 is 50, key is test_row_0/C:col10/1731680196018/Put/seqid=0 2024-11-15T14:16:36,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742277_1453 (size=9657) 2024-11-15T14:16:36,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/0e0082c7b7ca435380cad96a1dbff365 2024-11-15T14:16:36,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/6ec9127757244dc097397549e3dd09d5 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/6ec9127757244dc097397549e3dd09d5 2024-11-15T14:16:36,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/6ec9127757244dc097397549e3dd09d5, entries=200, sequenceid=58, filesize=14.0 K 2024-11-15T14:16:36,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/79c02d9c4f5c44f8b38e6507e86b0aa8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/79c02d9c4f5c44f8b38e6507e86b0aa8 2024-11-15T14:16:36,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/79c02d9c4f5c44f8b38e6507e86b0aa8, entries=100, sequenceid=58, filesize=9.4 K 2024-11-15T14:16:36,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/0e0082c7b7ca435380cad96a1dbff365 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0e0082c7b7ca435380cad96a1dbff365 2024-11-15T14:16:36,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680256192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0e0082c7b7ca435380cad96a1dbff365, entries=100, sequenceid=58, filesize=9.4 K 2024-11-15T14:16:36,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f24e805c31c506536abbd73db35b0c0f in 190ms, sequenceid=58, compaction requested=true 2024-11-15T14:16:36,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:36,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:36,209 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:36,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:36,210 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:36,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:36,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:36,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:36,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:36,211 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:36,211 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43019 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:36,211 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/A is initiating minor compaction (all files) 2024-11-15T14:16:36,211 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/B is initiating minor compaction (all files) 2024-11-15T14:16:36,211 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/B in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:36,211 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/A in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:36,211 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/562e143aa02f4750a96a57f5e478e7c6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/34a57699e0d2456fae745b9aeb5971b3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/79c02d9c4f5c44f8b38e6507e86b0aa8] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=32.9 K 2024-11-15T14:16:36,211 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a279978ce3284751b5a7a86df67d24ce, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/6eabb2a05ef149aeac10b30d0566c72d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/6ec9127757244dc097397549e3dd09d5] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=42.0 K 2024-11-15T14:16:36,212 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a279978ce3284751b5a7a86df67d24ce, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731680194488 2024-11-15T14:16:36,212 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 562e143aa02f4750a96a57f5e478e7c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731680194488 2024-11-15T14:16:36,212 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eabb2a05ef149aeac10b30d0566c72d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731680194640 2024-11-15T14:16:36,212 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 34a57699e0d2456fae745b9aeb5971b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731680194640 2024-11-15T14:16:36,212 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ec9127757244dc097397549e3dd09d5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731680195899 2024-11-15T14:16:36,212 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 79c02d9c4f5c44f8b38e6507e86b0aa8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731680195899 2024-11-15T14:16:36,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731680196070, max timestamp=1731680196213Num uniques 859; 2024-11-15T14:16:36,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T14:16:36,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-15T14:16:36,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:36,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:36,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:36,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:36,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:36,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:36,233 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#B#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:36,234 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/412125508dc94702839db5f80b3842f4 is 50, key is test_row_0/B:col10/1731680196018/Put/seqid=0 2024-11-15T14:16:36,243 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#A#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:36,244 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/12ec9e7800974f0a84b5184819702ad6 is 50, key is test_row_0/A:col10/1731680196018/Put/seqid=0 2024-11-15T14:16:36,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/b1c637bb3b3e42cda3d3765672848776 is 50, key is test_row_0/A:col10/1731680196213/Put/seqid=0 2024-11-15T14:16:36,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742279_1455 (size=12104) 2024-11-15T14:16:36,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742278_1454 (size=12104) 2024-11-15T14:16:36,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742280_1456 (size=14341) 2024-11-15T14:16:36,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/b1c637bb3b3e42cda3d3765672848776 2024-11-15T14:16:36,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/f5ec5505de3645ea879778027aafceae is 50, key is test_row_0/B:col10/1731680196213/Put/seqid=0 2024-11-15T14:16:36,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680256296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680256301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680256326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742281_1457 (size=12001) 2024-11-15T14:16:36,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680256412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680256432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680256432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680256452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T14:16:36,652 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:36,655 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-15T14:16:36,657 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T14:16:36,659 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:36,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:36,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680256652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680256658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680256672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,705 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/12ec9e7800974f0a84b5184819702ad6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/12ec9e7800974f0a84b5184819702ad6 2024-11-15T14:16:36,709 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/A of f24e805c31c506536abbd73db35b0c0f into 12ec9e7800974f0a84b5184819702ad6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:36,709 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:36,709 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/A, priority=13, startTime=1731680196209; duration=0sec 2024-11-15T14:16:36,709 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:36,709 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:A 2024-11-15T14:16:36,709 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:36,710 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:36,710 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/C is initiating minor compaction (all files) 2024-11-15T14:16:36,710 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/C in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:36,710 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/9d6497216bbf44d7b9fa5da2e73a56c4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e1386746a6d04ee9a9462e8f1997684e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0e0082c7b7ca435380cad96a1dbff365] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=32.9 K 2024-11-15T14:16:36,711 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d6497216bbf44d7b9fa5da2e73a56c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731680194488 2024-11-15T14:16:36,711 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1386746a6d04ee9a9462e8f1997684e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731680194640 2024-11-15T14:16:36,711 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e0082c7b7ca435380cad96a1dbff365, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731680195899 2024-11-15T14:16:36,728 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/412125508dc94702839db5f80b3842f4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/412125508dc94702839db5f80b3842f4 2024-11-15T14:16:36,729 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#C#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:36,729 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/863c098aea4145c5b05ea84aa5374be2 is 50, key is test_row_0/C:col10/1731680196018/Put/seqid=0 2024-11-15T14:16:36,737 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/B of f24e805c31c506536abbd73db35b0c0f into 412125508dc94702839db5f80b3842f4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:36,737 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:36,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,737 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/B, priority=13, startTime=1731680196209; duration=0sec 2024-11-15T14:16:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680256722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,738 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:36,738 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:B 2024-11-15T14:16:36,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/f5ec5505de3645ea879778027aafceae 2024-11-15T14:16:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T14:16:36,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742282_1458 (size=12104) 2024-11-15T14:16:36,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/2ff6b3d0afc94f58ab46bcb0738f3150 is 50, key is test_row_0/C:col10/1731680196213/Put/seqid=0 2024-11-15T14:16:36,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T14:16:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:36,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:36,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742283_1459 (size=12001) 2024-11-15T14:16:36,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680256911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,923 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in f24e805c31c506536abbd73db35b0c0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:16:36,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T14:16:36,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T14:16:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:36,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:36,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680256971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:36,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:36,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680256982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:37,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:37,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680256992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:37,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T14:16:37,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:37,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:37,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:37,188 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/863c098aea4145c5b05ea84aa5374be2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/863c098aea4145c5b05ea84aa5374be2 2024-11-15T14:16:37,197 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/C of f24e805c31c506536abbd73db35b0c0f into 863c098aea4145c5b05ea84aa5374be2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:37,198 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:37,198 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/C, priority=13, startTime=1731680196210; duration=0sec 2024-11-15T14:16:37,199 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:37,199 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:C 2024-11-15T14:16:37,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680257242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:37,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/2ff6b3d0afc94f58ab46bcb0738f3150 2024-11-15T14:16:37,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/b1c637bb3b3e42cda3d3765672848776 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/b1c637bb3b3e42cda3d3765672848776 2024-11-15T14:16:37,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/b1c637bb3b3e42cda3d3765672848776, entries=200, sequenceid=79, filesize=14.0 K 2024-11-15T14:16:37,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/f5ec5505de3645ea879778027aafceae as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/f5ec5505de3645ea879778027aafceae 2024-11-15T14:16:37,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T14:16:37,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:37,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:37,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:37,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:37,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T14:16:37,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/f5ec5505de3645ea879778027aafceae, entries=150, sequenceid=79, filesize=11.7 K 2024-11-15T14:16:37,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/2ff6b3d0afc94f58ab46bcb0738f3150 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/2ff6b3d0afc94f58ab46bcb0738f3150 2024-11-15T14:16:37,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/2ff6b3d0afc94f58ab46bcb0738f3150, entries=150, sequenceid=79, filesize=11.7 K 2024-11-15T14:16:37,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for f24e805c31c506536abbd73db35b0c0f in 1076ms, sequenceid=79, compaction requested=false 2024-11-15T14:16:37,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:37,398 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T14:16:37,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T14:16:37,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:37,432 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T14:16:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:37,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/ff24a7e4cfe346ed8cb3e83b72ae4f9f is 50, key is test_row_0/A:col10/1731680196290/Put/seqid=0 2024-11-15T14:16:37,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742284_1460 (size=12001) 2024-11-15T14:16:37,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:37,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680257536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:37,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680257546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680257547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:37,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680257663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:37,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680257663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680257663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T14:16:37,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680257872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680257872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:37,891 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/ff24a7e4cfe346ed8cb3e83b72ae4f9f 2024-11-15T14:16:37,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680257881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:37,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/66533c786ba749cc88d5b3f592c0ea91 is 50, key is test_row_0/B:col10/1731680196290/Put/seqid=0 2024-11-15T14:16:37,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742285_1461 (size=12001) 2024-11-15T14:16:38,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680258181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680258182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:38,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680258212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:38,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680258261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:38,284 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in f24e805c31c506536abbd73db35b0c0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:16:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:38,366 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/66533c786ba749cc88d5b3f592c0ea91 2024-11-15T14:16:38,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/b6607b492ae449b38858668d6eb199cf is 50, key is test_row_0/C:col10/1731680196290/Put/seqid=0 2024-11-15T14:16:38,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742286_1462 (size=12001) 2024-11-15T14:16:38,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680258701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680258702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:38,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680258733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T14:16:38,821 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/b6607b492ae449b38858668d6eb199cf 2024-11-15T14:16:38,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/ff24a7e4cfe346ed8cb3e83b72ae4f9f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ff24a7e4cfe346ed8cb3e83b72ae4f9f 2024-11-15T14:16:38,837 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ff24a7e4cfe346ed8cb3e83b72ae4f9f, entries=150, sequenceid=98, filesize=11.7 K 2024-11-15T14:16:38,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/66533c786ba749cc88d5b3f592c0ea91 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/66533c786ba749cc88d5b3f592c0ea91 2024-11-15T14:16:38,845 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/66533c786ba749cc88d5b3f592c0ea91, entries=150, sequenceid=98, filesize=11.7 K 2024-11-15T14:16:38,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/b6607b492ae449b38858668d6eb199cf as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b6607b492ae449b38858668d6eb199cf 2024-11-15T14:16:38,849 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b6607b492ae449b38858668d6eb199cf, entries=150, sequenceid=98, filesize=11.7 K 2024-11-15T14:16:38,851 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f24e805c31c506536abbd73db35b0c0f in 1419ms, sequenceid=98, compaction requested=true 2024-11-15T14:16:38,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:38,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:38,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-15T14:16:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-15T14:16:38,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-15T14:16:38,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1940 sec 2024-11-15T14:16:38,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.2010 sec 2024-11-15T14:16:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:38,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T14:16:38,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:38,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:38,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:38,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:38,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:38,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:38,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/a2eeb0b2d3e848fc96fa2c7f3459f3e9 is 50, key is test_row_0/A:col10/1731680197543/Put/seqid=0 2024-11-15T14:16:39,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742287_1463 (size=14341) 2024-11-15T14:16:39,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680259076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:39,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680259203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:39,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/a2eeb0b2d3e848fc96fa2c7f3459f3e9 2024-11-15T14:16:39,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/8a8ec3f4c7e64508b2463969c5cd31ec is 50, key is test_row_0/B:col10/1731680197543/Put/seqid=0 2024-11-15T14:16:39,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680259423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:39,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742288_1464 (size=12001) 2024-11-15T14:16:39,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/8a8ec3f4c7e64508b2463969c5cd31ec 2024-11-15T14:16:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:39,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/0d0cce1026d94c0982e20b5945ba65e4 is 50, key is test_row_0/C:col10/1731680197543/Put/seqid=0 2024-11-15T14:16:39,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742289_1465 (size=12001) 2024-11-15T14:16:39,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680259722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:39,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680259723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:39,726 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in f24e805c31c506536abbd73db35b0c0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:16:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:39,727 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in f24e805c31c506536abbd73db35b0c0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:16:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680259751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680259751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:39,761 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in f24e805c31c506536abbd73db35b0c0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:16:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:39,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/0d0cce1026d94c0982e20b5945ba65e4 2024-11-15T14:16:39,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/a2eeb0b2d3e848fc96fa2c7f3459f3e9 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a2eeb0b2d3e848fc96fa2c7f3459f3e9 2024-11-15T14:16:39,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a2eeb0b2d3e848fc96fa2c7f3459f3e9, entries=200, sequenceid=115, filesize=14.0 K 2024-11-15T14:16:39,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/8a8ec3f4c7e64508b2463969c5cd31ec as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/8a8ec3f4c7e64508b2463969c5cd31ec 2024-11-15T14:16:39,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/8a8ec3f4c7e64508b2463969c5cd31ec, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T14:16:39,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/0d0cce1026d94c0982e20b5945ba65e4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0d0cce1026d94c0982e20b5945ba65e4 2024-11-15T14:16:39,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0d0cce1026d94c0982e20b5945ba65e4, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T14:16:39,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f24e805c31c506536abbd73db35b0c0f in 975ms, sequenceid=115, compaction requested=true 2024-11-15T14:16:39,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:39,934 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:16:39,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:39,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:39,935 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:16:39,935 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:16:39,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:39,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:39,935 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/A is initiating minor compaction (all files) 2024-11-15T14:16:39,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:39,935 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/A in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:39,935 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/12ec9e7800974f0a84b5184819702ad6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/b1c637bb3b3e42cda3d3765672848776, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ff24a7e4cfe346ed8cb3e83b72ae4f9f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a2eeb0b2d3e848fc96fa2c7f3459f3e9] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=51.5 K 2024-11-15T14:16:39,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:39,936 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:16:39,936 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/B is initiating minor compaction (all files) 2024-11-15T14:16:39,936 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/B in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:39,936 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/412125508dc94702839db5f80b3842f4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/f5ec5505de3645ea879778027aafceae, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/66533c786ba749cc88d5b3f592c0ea91, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/8a8ec3f4c7e64508b2463969c5cd31ec] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=47.0 K 2024-11-15T14:16:39,936 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12ec9e7800974f0a84b5184819702ad6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731680194654 2024-11-15T14:16:39,937 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 412125508dc94702839db5f80b3842f4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731680194654 2024-11-15T14:16:39,938 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting f5ec5505de3645ea879778027aafceae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731680196070 2024-11-15T14:16:39,938 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1c637bb3b3e42cda3d3765672848776, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731680196070 2024-11-15T14:16:39,938 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff24a7e4cfe346ed8cb3e83b72ae4f9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731680196245 2024-11-15T14:16:39,938 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 66533c786ba749cc88d5b3f592c0ea91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731680196245 2024-11-15T14:16:39,939 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a8ec3f4c7e64508b2463969c5cd31ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731680197543 2024-11-15T14:16:39,939 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2eeb0b2d3e848fc96fa2c7f3459f3e9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731680197534 2024-11-15T14:16:39,953 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#B#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:39,954 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/35d2be05199b4443af9fa3702b256228 is 50, key is test_row_0/B:col10/1731680197543/Put/seqid=0 2024-11-15T14:16:39,959 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:39,960 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/451ef5dff3e640b4820b27e9fe98bc5e is 50, key is test_row_0/A:col10/1731680197543/Put/seqid=0 2024-11-15T14:16:39,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742290_1466 (size=12241) 2024-11-15T14:16:39,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742291_1467 (size=12241) 2024-11-15T14:16:40,018 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/35d2be05199b4443af9fa3702b256228 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/35d2be05199b4443af9fa3702b256228 2024-11-15T14:16:40,069 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/B of f24e805c31c506536abbd73db35b0c0f into 35d2be05199b4443af9fa3702b256228(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:40,069 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:40,069 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/B, priority=12, startTime=1731680199935; duration=0sec 2024-11-15T14:16:40,069 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:40,069 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:B 2024-11-15T14:16:40,069 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:16:40,088 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:16:40,088 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/C is initiating minor compaction (all files) 2024-11-15T14:16:40,088 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/C in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:40,088 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/863c098aea4145c5b05ea84aa5374be2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/2ff6b3d0afc94f58ab46bcb0738f3150, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b6607b492ae449b38858668d6eb199cf, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0d0cce1026d94c0982e20b5945ba65e4] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=47.0 K 2024-11-15T14:16:40,091 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 863c098aea4145c5b05ea84aa5374be2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731680194654 2024-11-15T14:16:40,095 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ff6b3d0afc94f58ab46bcb0738f3150, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731680196070 2024-11-15T14:16:40,098 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting b6607b492ae449b38858668d6eb199cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731680196245 2024-11-15T14:16:40,099 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d0cce1026d94c0982e20b5945ba65e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731680197543 2024-11-15T14:16:40,125 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#C#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T14:16:40,127 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/b5f5cee106074fc58098a8587cf1b0b7 is 50, key is test_row_0/C:col10/1731680197543/Put/seqid=0 2024-11-15T14:16:40,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742292_1468 (size=12241) 2024-11-15T14:16:40,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T14:16:40,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:40,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:40,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:40,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:40,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:40,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:40,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/1c5ce7c2f606494eb7bb936f15689293 is 50, key is test_row_0/A:col10/1731680199072/Put/seqid=0 2024-11-15T14:16:40,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742293_1469 (size=14441) 2024-11-15T14:16:40,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/1c5ce7c2f606494eb7bb936f15689293 2024-11-15T14:16:40,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/c37d69c50d2b4c579d89bcd1f7e8fc35 is 50, key is test_row_0/B:col10/1731680199072/Put/seqid=0 2024-11-15T14:16:40,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742294_1470 (size=12101) 2024-11-15T14:16:40,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/c37d69c50d2b4c579d89bcd1f7e8fc35 2024-11-15T14:16:40,403 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/451ef5dff3e640b4820b27e9fe98bc5e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/451ef5dff3e640b4820b27e9fe98bc5e 2024-11-15T14:16:40,413 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/A of f24e805c31c506536abbd73db35b0c0f into 451ef5dff3e640b4820b27e9fe98bc5e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:40,413 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:40,413 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/A, priority=12, startTime=1731680199934; duration=0sec 2024-11-15T14:16:40,413 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:40,413 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:A 2024-11-15T14:16:40,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/209091333d034fb5b1321c545221a2c7 is 50, key is test_row_0/C:col10/1731680199072/Put/seqid=0 2024-11-15T14:16:40,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680260419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:40,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680260419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:40,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742295_1471 (size=12101) 2024-11-15T14:16:40,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/209091333d034fb5b1321c545221a2c7 2024-11-15T14:16:40,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/1c5ce7c2f606494eb7bb936f15689293 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/1c5ce7c2f606494eb7bb936f15689293 2024-11-15T14:16:40,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/1c5ce7c2f606494eb7bb936f15689293, entries=200, sequenceid=132, filesize=14.1 K 2024-11-15T14:16:40,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/c37d69c50d2b4c579d89bcd1f7e8fc35 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c37d69c50d2b4c579d89bcd1f7e8fc35 2024-11-15T14:16:40,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c37d69c50d2b4c579d89bcd1f7e8fc35, entries=150, sequenceid=132, filesize=11.8 K 2024-11-15T14:16:40,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/209091333d034fb5b1321c545221a2c7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/209091333d034fb5b1321c545221a2c7 2024-11-15T14:16:40,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/209091333d034fb5b1321c545221a2c7, entries=150, sequenceid=132, filesize=11.8 K 2024-11-15T14:16:40,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for f24e805c31c506536abbd73db35b0c0f in 198ms, sequenceid=132, compaction requested=false 2024-11-15T14:16:40,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:40,481 DEBUG [regionserver/26754e923c4a:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.15384615384615385, tune throughput to 57.69 MB/second 2024-11-15T14:16:40,493 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a279978ce3284751b5a7a86df67d24ce, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/6eabb2a05ef149aeac10b30d0566c72d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/6ec9127757244dc097397549e3dd09d5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/12ec9e7800974f0a84b5184819702ad6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/b1c637bb3b3e42cda3d3765672848776, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ff24a7e4cfe346ed8cb3e83b72ae4f9f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a2eeb0b2d3e848fc96fa2c7f3459f3e9] to archive 2024-11-15T14:16:40,495 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:16:40,496 WARN [regionserver/26754e923c4a:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-15T14:16:40,501 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a279978ce3284751b5a7a86df67d24ce to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a279978ce3284751b5a7a86df67d24ce 2024-11-15T14:16:40,513 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/6eabb2a05ef149aeac10b30d0566c72d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/6eabb2a05ef149aeac10b30d0566c72d 2024-11-15T14:16:40,515 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/6ec9127757244dc097397549e3dd09d5 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/6ec9127757244dc097397549e3dd09d5 2024-11-15T14:16:40,517 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/12ec9e7800974f0a84b5184819702ad6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/12ec9e7800974f0a84b5184819702ad6 2024-11-15T14:16:40,518 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/b1c637bb3b3e42cda3d3765672848776 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/b1c637bb3b3e42cda3d3765672848776 2024-11-15T14:16:40,521 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ff24a7e4cfe346ed8cb3e83b72ae4f9f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ff24a7e4cfe346ed8cb3e83b72ae4f9f 2024-11-15T14:16:40,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a2eeb0b2d3e848fc96fa2c7f3459f3e9 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a2eeb0b2d3e848fc96fa2c7f3459f3e9 2024-11-15T14:16:40,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/562e143aa02f4750a96a57f5e478e7c6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/34a57699e0d2456fae745b9aeb5971b3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/412125508dc94702839db5f80b3842f4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/79c02d9c4f5c44f8b38e6507e86b0aa8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/f5ec5505de3645ea879778027aafceae, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/66533c786ba749cc88d5b3f592c0ea91, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/8a8ec3f4c7e64508b2463969c5cd31ec] to archive 2024-11-15T14:16:40,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:16:40,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/562e143aa02f4750a96a57f5e478e7c6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/562e143aa02f4750a96a57f5e478e7c6 2024-11-15T14:16:40,535 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/34a57699e0d2456fae745b9aeb5971b3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/34a57699e0d2456fae745b9aeb5971b3 2024-11-15T14:16:40,537 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/412125508dc94702839db5f80b3842f4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/412125508dc94702839db5f80b3842f4 2024-11-15T14:16:40,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/79c02d9c4f5c44f8b38e6507e86b0aa8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/79c02d9c4f5c44f8b38e6507e86b0aa8 2024-11-15T14:16:40,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/f5ec5505de3645ea879778027aafceae to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/f5ec5505de3645ea879778027aafceae 2024-11-15T14:16:40,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/66533c786ba749cc88d5b3f592c0ea91 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/66533c786ba749cc88d5b3f592c0ea91 2024-11-15T14:16:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:40,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T14:16:40,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:40,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:40,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/8a8ec3f4c7e64508b2463969c5cd31ec to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/8a8ec3f4c7e64508b2463969c5cd31ec 2024-11-15T14:16:40,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:40,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:40,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:40,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:40,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/9d6497216bbf44d7b9fa5da2e73a56c4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e1386746a6d04ee9a9462e8f1997684e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0e0082c7b7ca435380cad96a1dbff365] to archive 2024-11-15T14:16:40,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:16:40,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/9d6497216bbf44d7b9fa5da2e73a56c4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/9d6497216bbf44d7b9fa5da2e73a56c4 2024-11-15T14:16:40,556 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e1386746a6d04ee9a9462e8f1997684e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e1386746a6d04ee9a9462e8f1997684e 2024-11-15T14:16:40,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/ddc62289047d407b9a0780370f7129c4 is 50, key is test_row_0/A:col10/1731680200547/Put/seqid=0 2024-11-15T14:16:40,559 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/26754e923c4a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0e0082c7b7ca435380cad96a1dbff365 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0e0082c7b7ca435380cad96a1dbff365 2024-11-15T14:16:40,559 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/b5f5cee106074fc58098a8587cf1b0b7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b5f5cee106074fc58098a8587cf1b0b7 2024-11-15T14:16:40,568 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/C of f24e805c31c506536abbd73db35b0c0f into b5f5cee106074fc58098a8587cf1b0b7(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:40,569 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:40,569 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/C, priority=12, startTime=1731680199935; duration=0sec 2024-11-15T14:16:40,569 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:40,569 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:C 2024-11-15T14:16:40,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742296_1472 (size=14541) 2024-11-15T14:16:40,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/ddc62289047d407b9a0780370f7129c4 2024-11-15T14:16:40,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/62e5e86d1cf34f42a52c84cba1039814 is 50, key is test_row_0/B:col10/1731680200547/Put/seqid=0 2024-11-15T14:16:40,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680260597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:40,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680260606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:40,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742297_1473 (size=12151) 2024-11-15T14:16:40,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680260722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:40,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:40,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680260732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T14:16:40,812 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:40,823 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-15T14:16:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T14:16:40,825 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:40,825 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:40,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T14:16:40,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680260933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:40,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680260952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:40,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T14:16:40,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:40,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:40,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:40,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:40,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/62e5e86d1cf34f42a52c84cba1039814 2024-11-15T14:16:41,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/7fc382c553c243cc874df8b92612a361 is 50, key is test_row_0/C:col10/1731680200547/Put/seqid=0 2024-11-15T14:16:41,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742298_1474 (size=12151) 2024-11-15T14:16:41,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/7fc382c553c243cc874df8b92612a361 2024-11-15T14:16:41,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/ddc62289047d407b9a0780370f7129c4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ddc62289047d407b9a0780370f7129c4 2024-11-15T14:16:41,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ddc62289047d407b9a0780370f7129c4, entries=200, sequenceid=151, filesize=14.2 K 2024-11-15T14:16:41,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/62e5e86d1cf34f42a52c84cba1039814 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/62e5e86d1cf34f42a52c84cba1039814 2024-11-15T14:16:41,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/62e5e86d1cf34f42a52c84cba1039814, entries=150, sequenceid=151, filesize=11.9 K 2024-11-15T14:16:41,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/7fc382c553c243cc874df8b92612a361 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/7fc382c553c243cc874df8b92612a361 2024-11-15T14:16:41,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T14:16:41,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:41,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T14:16:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/7fc382c553c243cc874df8b92612a361, entries=150, sequenceid=151, filesize=11.9 K 2024-11-15T14:16:41,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f24e805c31c506536abbd73db35b0c0f in 632ms, sequenceid=151, compaction requested=true 2024-11-15T14:16:41,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:41,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:41,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:41,182 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:41,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:41,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:41,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:41,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:16:41,182 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:41,182 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41223 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:41,182 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:41,183 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/A is initiating minor compaction (all files) 2024-11-15T14:16:41,183 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/B is initiating minor compaction (all files) 2024-11-15T14:16:41,183 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/A in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,183 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/B in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,183 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/451ef5dff3e640b4820b27e9fe98bc5e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/1c5ce7c2f606494eb7bb936f15689293, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ddc62289047d407b9a0780370f7129c4] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=40.3 K 2024-11-15T14:16:41,183 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/35d2be05199b4443af9fa3702b256228, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c37d69c50d2b4c579d89bcd1f7e8fc35, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/62e5e86d1cf34f42a52c84cba1039814] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=35.6 K 2024-11-15T14:16:41,183 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 451ef5dff3e640b4820b27e9fe98bc5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731680197543 2024-11-15T14:16:41,183 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 35d2be05199b4443af9fa3702b256228, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731680197543 2024-11-15T14:16:41,183 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c37d69c50d2b4c579d89bcd1f7e8fc35, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731680199039 2024-11-15T14:16:41,183 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c5ce7c2f606494eb7bb936f15689293, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731680199032 2024-11-15T14:16:41,183 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 62e5e86d1cf34f42a52c84cba1039814, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731680200379 2024-11-15T14:16:41,184 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddc62289047d407b9a0780370f7129c4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731680200379 2024-11-15T14:16:41,202 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#A#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:41,203 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/1b0039cf39b34d3aafc6f98dd4871372 is 50, key is test_row_0/A:col10/1731680200547/Put/seqid=0 2024-11-15T14:16:41,213 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#B#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:41,213 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/4b9f909f254440f89acd7aae3c7328f0 is 50, key is test_row_0/B:col10/1731680200547/Put/seqid=0 2024-11-15T14:16:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:41,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T14:16:41,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:41,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:41,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:41,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:41,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:41,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:41,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742299_1475 (size=12254) 2024-11-15T14:16:41,276 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/1b0039cf39b34d3aafc6f98dd4871372 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/1b0039cf39b34d3aafc6f98dd4871372 2024-11-15T14:16:41,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742300_1476 (size=12254) 2024-11-15T14:16:41,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/3878643e6eeb4215bc9887dd9edaf850 is 50, key is test_row_0/A:col10/1731680200601/Put/seqid=0 2024-11-15T14:16:41,288 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/A of f24e805c31c506536abbd73db35b0c0f into 1b0039cf39b34d3aafc6f98dd4871372(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:41,289 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:41,289 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/A, priority=13, startTime=1731680201181; duration=0sec 2024-11-15T14:16:41,289 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:41,289 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:A 2024-11-15T14:16:41,289 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:41,297 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:41,297 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/C is initiating minor compaction (all files) 2024-11-15T14:16:41,297 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/C in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,297 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b5f5cee106074fc58098a8587cf1b0b7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/209091333d034fb5b1321c545221a2c7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/7fc382c553c243cc874df8b92612a361] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=35.6 K 2024-11-15T14:16:41,297 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5f5cee106074fc58098a8587cf1b0b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731680197543 2024-11-15T14:16:41,298 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 209091333d034fb5b1321c545221a2c7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731680199039 2024-11-15T14:16:41,298 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fc382c553c243cc874df8b92612a361, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731680200379 2024-11-15T14:16:41,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T14:16:41,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:41,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,313 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:41,314 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/0eddae22ec9c43069915069f1a50a6d6 is 50, key is test_row_0/C:col10/1731680200547/Put/seqid=0 2024-11-15T14:16:41,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742301_1477 (size=14541) 2024-11-15T14:16:41,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742302_1478 (size=12391) 2024-11-15T14:16:41,356 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/0eddae22ec9c43069915069f1a50a6d6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0eddae22ec9c43069915069f1a50a6d6 2024-11-15T14:16:41,368 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/C of f24e805c31c506536abbd73db35b0c0f into 0eddae22ec9c43069915069f1a50a6d6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:41,368 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:41,368 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/C, priority=13, startTime=1731680201182; duration=0sec 2024-11-15T14:16:41,368 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:41,368 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:C 2024-11-15T14:16:41,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680261376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:41,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680261378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T14:16:41,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T14:16:41,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:41,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680261492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:41,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:41,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680261502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:41,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T14:16:41,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:41,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,692 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/4b9f909f254440f89acd7aae3c7328f0 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/4b9f909f254440f89acd7aae3c7328f0 2024-11-15T14:16:41,697 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/B of f24e805c31c506536abbd73db35b0c0f into 4b9f909f254440f89acd7aae3c7328f0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:41,697 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:41,697 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/B, priority=13, startTime=1731680201182; duration=0sec 2024-11-15T14:16:41,697 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:41,697 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:B 2024-11-15T14:16:41,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680261711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:41,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680261722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:41,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/3878643e6eeb4215bc9887dd9edaf850 2024-11-15T14:16:41,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680261742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:41,749 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in f24e805c31c506536abbd73db35b0c0f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:16:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:41,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T14:16:41,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:41,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680261753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,761 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in f24e805c31c506536abbd73db35b0c0f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:16:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:41,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/cc2339794c2a47568d9d275bede609f8 is 50, key is test_row_0/B:col10/1731680200601/Put/seqid=0 2024-11-15T14:16:41,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680261771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:41,780 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in f24e805c31c506536abbd73db35b0c0f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:16:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:41,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742303_1479 (size=12151) 2024-11-15T14:16:41,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/cc2339794c2a47568d9d275bede609f8 2024-11-15T14:16:41,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/340327dd8df44ac084c75369fefbbca7 is 50, key is test_row_0/C:col10/1731680200601/Put/seqid=0 2024-11-15T14:16:41,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742304_1480 (size=12151) 2024-11-15T14:16:41,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/340327dd8df44ac084c75369fefbbca7 2024-11-15T14:16:41,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/3878643e6eeb4215bc9887dd9edaf850 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/3878643e6eeb4215bc9887dd9edaf850 2024-11-15T14:16:41,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/3878643e6eeb4215bc9887dd9edaf850, entries=200, sequenceid=167, filesize=14.2 K 2024-11-15T14:16:41,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/cc2339794c2a47568d9d275bede609f8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/cc2339794c2a47568d9d275bede609f8 2024-11-15T14:16:41,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/cc2339794c2a47568d9d275bede609f8, entries=150, sequenceid=167, filesize=11.9 K 2024-11-15T14:16:41,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/340327dd8df44ac084c75369fefbbca7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/340327dd8df44ac084c75369fefbbca7 2024-11-15T14:16:41,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T14:16:41,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:41,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:41,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:41,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/340327dd8df44ac084c75369fefbbca7, entries=150, sequenceid=167, filesize=11.9 K 2024-11-15T14:16:41,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f24e805c31c506536abbd73db35b0c0f in 655ms, sequenceid=167, compaction requested=false 2024-11-15T14:16:41,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T14:16:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:42,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T14:16:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:42,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:42,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/21f28a04b2b640a2bcaf4ba89d67c3be is 50, key is test_row_0/A:col10/1731680201376/Put/seqid=0 2024-11-15T14:16:42,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T14:16:42,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:42,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:42,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:42,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:42,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:42,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742305_1481 (size=14541) 2024-11-15T14:16:42,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/21f28a04b2b640a2bcaf4ba89d67c3be 2024-11-15T14:16:42,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/ecedb12e00e6426a9a24ca25b9cbbf61 is 50, key is test_row_0/B:col10/1731680201376/Put/seqid=0 2024-11-15T14:16:42,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742306_1482 (size=12151) 2024-11-15T14:16:42,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/ecedb12e00e6426a9a24ca25b9cbbf61 2024-11-15T14:16:42,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680262115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:42,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:42,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680262123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:42,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/0cc48cb5cee54b49b254895ec08f377e is 50, key is test_row_0/C:col10/1731680201376/Put/seqid=0 2024-11-15T14:16:42,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742307_1483 (size=12151) 2024-11-15T14:16:42,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T14:16:42,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:42,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:42,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:42,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:42,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:42,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680262241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680262241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:42,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T14:16:42,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:42,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:42,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:42,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:42,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:42,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680262452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:42,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680262452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:42,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T14:16:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:42,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:42,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:42,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:42,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:42,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/0cc48cb5cee54b49b254895ec08f377e 2024-11-15T14:16:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/21f28a04b2b640a2bcaf4ba89d67c3be as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/21f28a04b2b640a2bcaf4ba89d67c3be 2024-11-15T14:16:42,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/21f28a04b2b640a2bcaf4ba89d67c3be, entries=200, sequenceid=188, filesize=14.2 K 2024-11-15T14:16:42,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/ecedb12e00e6426a9a24ca25b9cbbf61 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/ecedb12e00e6426a9a24ca25b9cbbf61 2024-11-15T14:16:42,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/ecedb12e00e6426a9a24ca25b9cbbf61, entries=150, sequenceid=188, filesize=11.9 K 2024-11-15T14:16:42,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/0cc48cb5cee54b49b254895ec08f377e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0cc48cb5cee54b49b254895ec08f377e 2024-11-15T14:16:42,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0cc48cb5cee54b49b254895ec08f377e, entries=150, sequenceid=188, filesize=11.9 K 2024-11-15T14:16:42,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f24e805c31c506536abbd73db35b0c0f in 614ms, sequenceid=188, compaction requested=true 2024-11-15T14:16:42,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:42,647 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:42,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:42,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:42,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:42,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:42,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:42,647 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:42,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:42,648 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41336 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:42,648 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/A is initiating minor compaction (all files) 2024-11-15T14:16:42,648 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/A in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:42,648 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/1b0039cf39b34d3aafc6f98dd4871372, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/3878643e6eeb4215bc9887dd9edaf850, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/21f28a04b2b640a2bcaf4ba89d67c3be] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=40.4 K 2024-11-15T14:16:42,649 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b0039cf39b34d3aafc6f98dd4871372, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731680200379 2024-11-15T14:16:42,649 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36556 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:42,649 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/B is initiating minor compaction (all files) 2024-11-15T14:16:42,649 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/B in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:42,649 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3878643e6eeb4215bc9887dd9edaf850, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731680200563 2024-11-15T14:16:42,649 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/4b9f909f254440f89acd7aae3c7328f0, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/cc2339794c2a47568d9d275bede609f8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/ecedb12e00e6426a9a24ca25b9cbbf61] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=35.7 K 2024-11-15T14:16:42,650 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b9f909f254440f89acd7aae3c7328f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731680200379 2024-11-15T14:16:42,651 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21f28a04b2b640a2bcaf4ba89d67c3be, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731680201326 2024-11-15T14:16:42,656 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting cc2339794c2a47568d9d275bede609f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731680200563 2024-11-15T14:16:42,656 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting ecedb12e00e6426a9a24ca25b9cbbf61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731680201359 2024-11-15T14:16:42,669 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#A#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:42,669 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/852a9c7841834e9d8e2054e4217da5f6 is 50, key is test_row_0/A:col10/1731680201376/Put/seqid=0 2024-11-15T14:16:42,672 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#B#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:42,674 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/2a31f172584a4fd4a3f6633dd2234fa5 is 50, key is test_row_0/B:col10/1731680201376/Put/seqid=0 2024-11-15T14:16:42,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T14:16:42,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:42,688 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T14:16:42,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:42,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:42,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:42,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:42,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:42,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:42,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742308_1484 (size=12357) 2024-11-15T14:16:42,699 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/852a9c7841834e9d8e2054e4217da5f6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/852a9c7841834e9d8e2054e4217da5f6 2024-11-15T14:16:42,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/60cc142a96934dcda7dd6a3a91de5c4e is 50, key is test_row_0/A:col10/1731680202113/Put/seqid=0 2024-11-15T14:16:42,710 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/A of f24e805c31c506536abbd73db35b0c0f into 852a9c7841834e9d8e2054e4217da5f6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:42,710 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:42,710 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/A, priority=13, startTime=1731680202647; duration=0sec 2024-11-15T14:16:42,710 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:42,710 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:A 2024-11-15T14:16:42,710 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:42,712 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:42,716 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/C is initiating minor compaction (all files) 2024-11-15T14:16:42,716 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/C in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:42,716 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0eddae22ec9c43069915069f1a50a6d6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/340327dd8df44ac084c75369fefbbca7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0cc48cb5cee54b49b254895ec08f377e] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=35.8 K 2024-11-15T14:16:42,720 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eddae22ec9c43069915069f1a50a6d6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731680200379 2024-11-15T14:16:42,721 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 340327dd8df44ac084c75369fefbbca7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731680200563 2024-11-15T14:16:42,721 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cc48cb5cee54b49b254895ec08f377e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731680201359 2024-11-15T14:16:42,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742309_1485 (size=12357) 2024-11-15T14:16:42,756 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:42,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742310_1486 (size=12151) 2024-11-15T14:16:42,757 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/101ed7b2c1844de993052f80ba9775f6 is 50, key is test_row_0/C:col10/1731680201376/Put/seqid=0 2024-11-15T14:16:42,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:42,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742311_1487 (size=12493) 2024-11-15T14:16:42,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680262836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:42,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680262838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:42,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680262951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:42,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680262962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T14:16:43,181 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/60cc142a96934dcda7dd6a3a91de5c4e 2024-11-15T14:16:43,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680263172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:43,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680263172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:43,189 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/2a31f172584a4fd4a3f6633dd2234fa5 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/2a31f172584a4fd4a3f6633dd2234fa5 2024-11-15T14:16:43,204 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/B of f24e805c31c506536abbd73db35b0c0f into 2a31f172584a4fd4a3f6633dd2234fa5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:43,204 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:43,204 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/B, priority=13, startTime=1731680202647; duration=0sec 2024-11-15T14:16:43,205 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:43,205 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:B 2024-11-15T14:16:43,212 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/101ed7b2c1844de993052f80ba9775f6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/101ed7b2c1844de993052f80ba9775f6 2024-11-15T14:16:43,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/c1755b20b5c8487da982095bb003ca6c is 50, key is test_row_0/B:col10/1731680202113/Put/seqid=0 2024-11-15T14:16:43,219 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/C of f24e805c31c506536abbd73db35b0c0f into 101ed7b2c1844de993052f80ba9775f6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:43,219 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:43,220 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/C, priority=13, startTime=1731680202647; duration=0sec 2024-11-15T14:16:43,220 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:43,220 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:C 2024-11-15T14:16:43,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742312_1488 (size=12151) 2024-11-15T14:16:43,248 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/c1755b20b5c8487da982095bb003ca6c 2024-11-15T14:16:43,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/8817a1c2dc294b198f637d8fd0d020e4 is 50, key is test_row_0/C:col10/1731680202113/Put/seqid=0 2024-11-15T14:16:43,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742313_1489 (size=12151) 2024-11-15T14:16:43,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680263492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:43,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680263492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:43,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:43,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:43,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:43,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:43,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:43,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:43,727 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/8817a1c2dc294b198f637d8fd0d020e4 2024-11-15T14:16:43,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/60cc142a96934dcda7dd6a3a91de5c4e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/60cc142a96934dcda7dd6a3a91de5c4e 2024-11-15T14:16:43,736 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/60cc142a96934dcda7dd6a3a91de5c4e, entries=150, sequenceid=202, filesize=11.9 K 2024-11-15T14:16:43,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/c1755b20b5c8487da982095bb003ca6c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c1755b20b5c8487da982095bb003ca6c 2024-11-15T14:16:43,745 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c1755b20b5c8487da982095bb003ca6c, entries=150, sequenceid=202, filesize=11.9 K 2024-11-15T14:16:43,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/8817a1c2dc294b198f637d8fd0d020e4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/8817a1c2dc294b198f637d8fd0d020e4 2024-11-15T14:16:43,759 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/8817a1c2dc294b198f637d8fd0d020e4, entries=150, sequenceid=202, filesize=11.9 K 2024-11-15T14:16:43,760 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f24e805c31c506536abbd73db35b0c0f in 1072ms, sequenceid=202, compaction requested=false 2024-11-15T14:16:43,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:43,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:43,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-15T14:16:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-15T14:16:43,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-15T14:16:43,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9370 sec 2024-11-15T14:16:43,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.9440 sec 2024-11-15T14:16:44,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T14:16:44,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:44,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:44,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:44,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:44,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:44,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:44,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/590357b28ecc46da9e1dc79eaec66adc is 50, key is test_row_0/A:col10/1731680202783/Put/seqid=0 2024-11-15T14:16:44,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742314_1490 (size=16931) 2024-11-15T14:16:44,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/590357b28ecc46da9e1dc79eaec66adc 2024-11-15T14:16:44,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/78bcbe97f0174c73bb8d0e8147d7462d is 50, key is test_row_0/B:col10/1731680202783/Put/seqid=0 2024-11-15T14:16:44,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680264096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:44,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680264096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:44,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:44,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742315_1491 (size=12151) 2024-11-15T14:16:44,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680264222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:44,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680264222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:44,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680264433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:44,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680264433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:44,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/78bcbe97f0174c73bb8d0e8147d7462d 2024-11-15T14:16:44,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/a18a8c9bc27e445195978da308e7ab57 is 50, key is test_row_0/C:col10/1731680202783/Put/seqid=0 2024-11-15T14:16:44,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742316_1492 (size=12151) 2024-11-15T14:16:44,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/a18a8c9bc27e445195978da308e7ab57 2024-11-15T14:16:44,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/590357b28ecc46da9e1dc79eaec66adc as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/590357b28ecc46da9e1dc79eaec66adc 2024-11-15T14:16:44,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/590357b28ecc46da9e1dc79eaec66adc, entries=250, sequenceid=225, filesize=16.5 K 2024-11-15T14:16:44,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/78bcbe97f0174c73bb8d0e8147d7462d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/78bcbe97f0174c73bb8d0e8147d7462d 2024-11-15T14:16:44,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/78bcbe97f0174c73bb8d0e8147d7462d, entries=150, sequenceid=225, filesize=11.9 K 2024-11-15T14:16:44,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/a18a8c9bc27e445195978da308e7ab57 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a18a8c9bc27e445195978da308e7ab57 2024-11-15T14:16:44,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a18a8c9bc27e445195978da308e7ab57, entries=150, sequenceid=225, filesize=11.9 K 2024-11-15T14:16:44,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f24e805c31c506536abbd73db35b0c0f in 584ms, sequenceid=225, compaction requested=true 2024-11-15T14:16:44,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:44,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:44,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:44,606 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:44,606 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:44,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:44,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:44,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:44,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:44,607 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:44,607 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/A is initiating minor compaction (all files) 2024-11-15T14:16:44,607 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/A in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:44,607 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/852a9c7841834e9d8e2054e4217da5f6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/60cc142a96934dcda7dd6a3a91de5c4e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/590357b28ecc46da9e1dc79eaec66adc] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=40.5 K 2024-11-15T14:16:44,608 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:44,608 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/B is initiating minor compaction (all files) 2024-11-15T14:16:44,608 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/B in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:44,608 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/2a31f172584a4fd4a3f6633dd2234fa5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c1755b20b5c8487da982095bb003ca6c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/78bcbe97f0174c73bb8d0e8147d7462d] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=35.8 K 2024-11-15T14:16:44,608 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a31f172584a4fd4a3f6633dd2234fa5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731680201359 2024-11-15T14:16:44,608 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 852a9c7841834e9d8e2054e4217da5f6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731680201359 2024-11-15T14:16:44,609 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60cc142a96934dcda7dd6a3a91de5c4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731680202072 2024-11-15T14:16:44,609 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c1755b20b5c8487da982095bb003ca6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731680202072 2024-11-15T14:16:44,609 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 590357b28ecc46da9e1dc79eaec66adc, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731680202783 2024-11-15T14:16:44,609 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 78bcbe97f0174c73bb8d0e8147d7462d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731680202783 2024-11-15T14:16:44,622 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:44,622 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#A#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:44,622 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/eb3a5731780e4f3f84c0e2a10b18e0a6 is 50, key is test_row_0/A:col10/1731680202783/Put/seqid=0 2024-11-15T14:16:44,624 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/ab6d2ecc67bc4b40856428264c237087 is 50, key is test_row_0/B:col10/1731680202783/Put/seqid=0 2024-11-15T14:16:44,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742317_1493 (size=12459) 2024-11-15T14:16:44,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742318_1494 (size=12459) 2024-11-15T14:16:44,663 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/eb3a5731780e4f3f84c0e2a10b18e0a6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/eb3a5731780e4f3f84c0e2a10b18e0a6 2024-11-15T14:16:44,668 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/ab6d2ecc67bc4b40856428264c237087 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/ab6d2ecc67bc4b40856428264c237087 2024-11-15T14:16:44,669 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/A of f24e805c31c506536abbd73db35b0c0f into eb3a5731780e4f3f84c0e2a10b18e0a6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:44,669 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:44,669 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/A, priority=13, startTime=1731680204606; duration=0sec 2024-11-15T14:16:44,669 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:44,669 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:A 2024-11-15T14:16:44,669 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:44,670 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:44,670 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/C is initiating minor compaction (all files) 2024-11-15T14:16:44,670 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/C in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:44,671 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/101ed7b2c1844de993052f80ba9775f6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/8817a1c2dc294b198f637d8fd0d020e4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a18a8c9bc27e445195978da308e7ab57] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=35.9 K 2024-11-15T14:16:44,671 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 101ed7b2c1844de993052f80ba9775f6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731680201359 2024-11-15T14:16:44,672 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8817a1c2dc294b198f637d8fd0d020e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731680202072 2024-11-15T14:16:44,672 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a18a8c9bc27e445195978da308e7ab57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731680202783 2024-11-15T14:16:44,674 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/B of f24e805c31c506536abbd73db35b0c0f into ab6d2ecc67bc4b40856428264c237087(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:44,674 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:44,674 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/B, priority=13, startTime=1731680204606; duration=0sec 2024-11-15T14:16:44,675 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:44,675 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:B 2024-11-15T14:16:44,705 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#C#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:44,706 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/2756d52112a34482b85e58fdf1610e54 is 50, key is test_row_0/C:col10/1731680202783/Put/seqid=0 2024-11-15T14:16:44,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742319_1495 (size=12595) 2024-11-15T14:16:44,755 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/2756d52112a34482b85e58fdf1610e54 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/2756d52112a34482b85e58fdf1610e54 2024-11-15T14:16:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:44,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T14:16:44,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:44,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:44,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:44,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:44,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:44,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:44,764 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/C of f24e805c31c506536abbd73db35b0c0f into 2756d52112a34482b85e58fdf1610e54(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:44,764 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:44,764 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/C, priority=13, startTime=1731680204606; duration=0sec 2024-11-15T14:16:44,764 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:44,764 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:C 2024-11-15T14:16:44,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/83a4f11b2b484e6fb2fe1c5b9ea47a26 is 50, key is test_row_0/A:col10/1731680204755/Put/seqid=0 2024-11-15T14:16:44,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742320_1496 (size=14541) 2024-11-15T14:16:44,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680264873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:44,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680264875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T14:16:44,983 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:44,983 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:44,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-15T14:16:44,986 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T14:16:44,987 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:44,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:45,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:45,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680264992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:45,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680264992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T14:16:45,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T14:16:45,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:45,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:45,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:45,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680265211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:45,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680265212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:45,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/83a4f11b2b484e6fb2fe1c5b9ea47a26 2024-11-15T14:16:45,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/587412ba4f074251b6c05e2ca6d88166 is 50, key is test_row_0/B:col10/1731680204755/Put/seqid=0 2024-11-15T14:16:45,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742321_1497 (size=12151) 2024-11-15T14:16:45,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/587412ba4f074251b6c05e2ca6d88166 2024-11-15T14:16:45,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/e3129a0f6d8c4b419b2803d1985f9d1e is 50, key is test_row_0/C:col10/1731680204755/Put/seqid=0 2024-11-15T14:16:45,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T14:16:45,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:45,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:45,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:45,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:45,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T14:16:45,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742322_1498 (size=12151) 2024-11-15T14:16:45,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T14:16:45,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:45,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:45,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:45,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:45,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:45,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680265523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:45,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680265532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:45,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T14:16:45,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:45,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:45,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:45,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:45,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T14:16:45,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/e3129a0f6d8c4b419b2803d1985f9d1e 2024-11-15T14:16:45,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/83a4f11b2b484e6fb2fe1c5b9ea47a26 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/83a4f11b2b484e6fb2fe1c5b9ea47a26 2024-11-15T14:16:45,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/83a4f11b2b484e6fb2fe1c5b9ea47a26, entries=200, sequenceid=240, filesize=14.2 K 2024-11-15T14:16:45,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/587412ba4f074251b6c05e2ca6d88166 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/587412ba4f074251b6c05e2ca6d88166 2024-11-15T14:16:45,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/587412ba4f074251b6c05e2ca6d88166, entries=150, sequenceid=240, filesize=11.9 K 2024-11-15T14:16:45,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/e3129a0f6d8c4b419b2803d1985f9d1e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e3129a0f6d8c4b419b2803d1985f9d1e 2024-11-15T14:16:45,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e3129a0f6d8c4b419b2803d1985f9d1e, entries=150, sequenceid=240, filesize=11.9 K 2024-11-15T14:16:45,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f24e805c31c506536abbd73db35b0c0f in 989ms, sequenceid=240, compaction requested=false 2024-11-15T14:16:45,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:45,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T14:16:45,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:45,759 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T14:16:45,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:45,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:45,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:45,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:45,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:45,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:45,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/0fdd984b4d63457494b7844a488c68e3 is 50, key is test_row_0/A:col10/1731680204852/Put/seqid=0 2024-11-15T14:16:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:45,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:45,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742323_1499 (size=12251) 2024-11-15T14:16:45,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680265832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:45,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680265842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:45,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:45,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:45,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680265843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:45,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:45,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680265954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:45,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680265955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:45,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:45,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680265971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:46,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680266042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:46,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:46,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680266063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T14:16:46,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680266182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,195 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/0fdd984b4d63457494b7844a488c68e3 2024-11-15T14:16:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:46,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680266182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:46,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680266192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:46,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:46,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/44c753ae2a9b49bb80df7a3c9f6ae7be is 50, key is test_row_0/B:col10/1731680204852/Put/seqid=0 2024-11-15T14:16:46,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742324_1500 (size=12251) 2024-11-15T14:16:46,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680266511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:46,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680266514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:46,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680266514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:46,659 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/44c753ae2a9b49bb80df7a3c9f6ae7be 2024-11-15T14:16:46,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/96c840fa71424d1ea338966fb90750da is 50, key is test_row_0/C:col10/1731680204852/Put/seqid=0 2024-11-15T14:16:46,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742325_1501 (size=12251) 2024-11-15T14:16:46,726 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/96c840fa71424d1ea338966fb90750da 2024-11-15T14:16:46,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/0fdd984b4d63457494b7844a488c68e3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/0fdd984b4d63457494b7844a488c68e3 2024-11-15T14:16:46,735 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/0fdd984b4d63457494b7844a488c68e3, entries=150, sequenceid=260, filesize=12.0 K 2024-11-15T14:16:46,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/44c753ae2a9b49bb80df7a3c9f6ae7be as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/44c753ae2a9b49bb80df7a3c9f6ae7be 2024-11-15T14:16:46,741 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/44c753ae2a9b49bb80df7a3c9f6ae7be, entries=150, sequenceid=260, filesize=12.0 K 2024-11-15T14:16:46,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/96c840fa71424d1ea338966fb90750da as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/96c840fa71424d1ea338966fb90750da 2024-11-15T14:16:46,747 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/96c840fa71424d1ea338966fb90750da, entries=150, sequenceid=260, filesize=12.0 K 2024-11-15T14:16:46,748 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f24e805c31c506536abbd73db35b0c0f in 989ms, sequenceid=260, compaction requested=true 2024-11-15T14:16:46,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:46,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:46,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-15T14:16:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-15T14:16:46,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-15T14:16:46,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7630 sec 2024-11-15T14:16:46,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.7700 sec 2024-11-15T14:16:47,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:47,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T14:16:47,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:47,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:47,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:47,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:47,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:47,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:47,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/f72e102cc0544982a6a1fc4ad52e0c5f is 50, key is test_row_0/A:col10/1731680207050/Put/seqid=0 2024-11-15T14:16:47,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742326_1502 (size=12301) 2024-11-15T14:16:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680267111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T14:16:47,133 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:47,134 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-15T14:16:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T14:16:47,137 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:47,138 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:47,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:47,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680267126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680267129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680267130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680267131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T14:16:47,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680267245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680267253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680267253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680267263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680267263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T14:16:47,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:47,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:47,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:47,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T14:16:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:47,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T14:16:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680267471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680267472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680267472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680267472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680267482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/f72e102cc0544982a6a1fc4ad52e0c5f 2024-11-15T14:16:47,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/3fa5a1b8441b46f899950bdd410aa2dd is 50, key is test_row_0/B:col10/1731680207050/Put/seqid=0 2024-11-15T14:16:47,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742327_1503 (size=12301) 2024-11-15T14:16:47,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T14:16:47,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:47,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:47,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:47,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T14:16:47,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T14:16:47,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:47,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:47,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:47,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680267802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680267802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680267802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680267803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680267802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:47,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T14:16:47,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:47,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:48,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/3fa5a1b8441b46f899950bdd410aa2dd 2024-11-15T14:16:48,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/127a0717b4e94d7f8777e35b48afea33 is 50, key is test_row_0/C:col10/1731680207050/Put/seqid=0 2024-11-15T14:16:48,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742328_1504 (size=12301) 2024-11-15T14:16:48,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T14:16:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:48,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:48,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/127a0717b4e94d7f8777e35b48afea33 2024-11-15T14:16:48,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/f72e102cc0544982a6a1fc4ad52e0c5f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/f72e102cc0544982a6a1fc4ad52e0c5f 2024-11-15T14:16:48,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/f72e102cc0544982a6a1fc4ad52e0c5f, entries=150, sequenceid=275, filesize=12.0 K 2024-11-15T14:16:48,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/3fa5a1b8441b46f899950bdd410aa2dd as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/3fa5a1b8441b46f899950bdd410aa2dd 2024-11-15T14:16:48,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/3fa5a1b8441b46f899950bdd410aa2dd, entries=150, sequenceid=275, filesize=12.0 K 2024-11-15T14:16:48,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/127a0717b4e94d7f8777e35b48afea33 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/127a0717b4e94d7f8777e35b48afea33 2024-11-15T14:16:48,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/127a0717b4e94d7f8777e35b48afea33, entries=150, sequenceid=275, filesize=12.0 K 2024-11-15T14:16:48,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f24e805c31c506536abbd73db35b0c0f in 1124ms, sequenceid=275, compaction requested=true 2024-11-15T14:16:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:48,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:48,182 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:16:48,183 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:16:48,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:16:48,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:48,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:48,184 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:16:48,184 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/A is initiating minor compaction (all files) 2024-11-15T14:16:48,184 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/A in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:48,184 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/eb3a5731780e4f3f84c0e2a10b18e0a6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/83a4f11b2b484e6fb2fe1c5b9ea47a26, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/0fdd984b4d63457494b7844a488c68e3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/f72e102cc0544982a6a1fc4ad52e0c5f] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=50.3 K 2024-11-15T14:16:48,185 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49162 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:16:48,185 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/B is initiating minor compaction (all files) 2024-11-15T14:16:48,185 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/B in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:48,185 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/ab6d2ecc67bc4b40856428264c237087, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/587412ba4f074251b6c05e2ca6d88166, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/44c753ae2a9b49bb80df7a3c9f6ae7be, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/3fa5a1b8441b46f899950bdd410aa2dd] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=48.0 K 2024-11-15T14:16:48,189 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6d2ecc67bc4b40856428264c237087, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731680202783 2024-11-15T14:16:48,189 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb3a5731780e4f3f84c0e2a10b18e0a6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731680202783 2024-11-15T14:16:48,189 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 587412ba4f074251b6c05e2ca6d88166, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731680204055 2024-11-15T14:16:48,190 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83a4f11b2b484e6fb2fe1c5b9ea47a26, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731680204055 2024-11-15T14:16:48,190 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fdd984b4d63457494b7844a488c68e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731680204852 2024-11-15T14:16:48,190 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 44c753ae2a9b49bb80df7a3c9f6ae7be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731680204852 2024-11-15T14:16:48,191 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fa5a1b8441b46f899950bdd410aa2dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731680205837 2024-11-15T14:16:48,191 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f72e102cc0544982a6a1fc4ad52e0c5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731680205837 2024-11-15T14:16:48,211 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#A#compaction#431 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:48,212 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/5efe79e454e64e8cb8dcaa84d300251c is 50, key is test_row_0/A:col10/1731680207050/Put/seqid=0 2024-11-15T14:16:48,219 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#B#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:48,220 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/48a25c2b80594e7bb2ef98c48e3e6d1e is 50, key is test_row_0/B:col10/1731680207050/Put/seqid=0 2024-11-15T14:16:48,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T14:16:48,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:48,239 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T14:16:48,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:48,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:48,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:48,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:48,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:48,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T14:16:48,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742329_1505 (size=12745) 2024-11-15T14:16:48,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/79501c102ee34a5998a8c17cdb668085 is 50, key is test_row_0/A:col10/1731680207128/Put/seqid=0 2024-11-15T14:16:48,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742330_1506 (size=12745) 2024-11-15T14:16:48,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742331_1507 (size=12301) 2024-11-15T14:16:48,340 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/79501c102ee34a5998a8c17cdb668085 2024-11-15T14:16:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:48,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:48,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/42cfc0c7ec4d462eba58e9f7cd0622f0 is 50, key is test_row_0/B:col10/1731680207128/Put/seqid=0 2024-11-15T14:16:48,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742332_1508 (size=12301) 2024-11-15T14:16:48,382 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/42cfc0c7ec4d462eba58e9f7cd0622f0 2024-11-15T14:16:48,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680268367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680268368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680268368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680268370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680268386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/6ad58383e37d4b3eb65349da24755527 is 50, key is test_row_0/C:col10/1731680207128/Put/seqid=0 2024-11-15T14:16:48,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742333_1509 (size=12301) 2024-11-15T14:16:48,437 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/6ad58383e37d4b3eb65349da24755527 2024-11-15T14:16:48,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/79501c102ee34a5998a8c17cdb668085 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/79501c102ee34a5998a8c17cdb668085 2024-11-15T14:16:48,451 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/79501c102ee34a5998a8c17cdb668085, entries=150, sequenceid=292, filesize=12.0 K 2024-11-15T14:16:48,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/42cfc0c7ec4d462eba58e9f7cd0622f0 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/42cfc0c7ec4d462eba58e9f7cd0622f0 2024-11-15T14:16:48,463 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/42cfc0c7ec4d462eba58e9f7cd0622f0, entries=150, sequenceid=292, filesize=12.0 K 2024-11-15T14:16:48,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/6ad58383e37d4b3eb65349da24755527 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/6ad58383e37d4b3eb65349da24755527 2024-11-15T14:16:48,470 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/6ad58383e37d4b3eb65349da24755527, entries=150, sequenceid=292, filesize=12.0 K 2024-11-15T14:16:48,471 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f24e805c31c506536abbd73db35b0c0f in 232ms, sequenceid=292, compaction requested=true 2024-11-15T14:16:48,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:48,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:48,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-15T14:16:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-15T14:16:48,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-15T14:16:48,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3350 sec 2024-11-15T14:16:48,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.3420 sec 2024-11-15T14:16:48,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T14:16:48,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:48,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:48,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:48,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:48,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:48,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:48,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/5fc0904288244a618bc7d1fd2e4e61b8 is 50, key is test_row_0/A:col10/1731680208384/Put/seqid=0 2024-11-15T14:16:48,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742334_1510 (size=14741) 2024-11-15T14:16:48,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680268550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680268551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680268553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680268553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680268571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680268672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680268681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680268681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680268681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,695 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/5efe79e454e64e8cb8dcaa84d300251c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/5efe79e454e64e8cb8dcaa84d300251c 2024-11-15T14:16:48,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680268692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,708 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/A of f24e805c31c506536abbd73db35b0c0f into 5efe79e454e64e8cb8dcaa84d300251c(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:48,708 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:48,708 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/A, priority=12, startTime=1731680208182; duration=0sec 2024-11-15T14:16:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,708 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,708 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:A 2024-11-15T14:16:48,709 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T14:16:48,710 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61599 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T14:16:48,710 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/C is initiating minor compaction (all files) 2024-11-15T14:16:48,710 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/C in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:48,710 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/2756d52112a34482b85e58fdf1610e54, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e3129a0f6d8c4b419b2803d1985f9d1e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/96c840fa71424d1ea338966fb90750da, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/127a0717b4e94d7f8777e35b48afea33, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/6ad58383e37d4b3eb65349da24755527] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=60.2 K 2024-11-15T14:16:48,711 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2756d52112a34482b85e58fdf1610e54, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731680202783 2024-11-15T14:16:48,711 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3129a0f6d8c4b419b2803d1985f9d1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731680204055 2024-11-15T14:16:48,711 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96c840fa71424d1ea338966fb90750da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731680204852 2024-11-15T14:16:48,711 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 127a0717b4e94d7f8777e35b48afea33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731680205837 2024-11-15T14:16:48,712 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ad58383e37d4b3eb65349da24755527, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731680207124 2024-11-15T14:16:48,720 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/48a25c2b80594e7bb2ef98c48e3e6d1e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/48a25c2b80594e7bb2ef98c48e3e6d1e 2024-11-15T14:16:48,734 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/B of f24e805c31c506536abbd73db35b0c0f into 48a25c2b80594e7bb2ef98c48e3e6d1e(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:48,734 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:48,734 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/B, priority=12, startTime=1731680208182; duration=0sec 2024-11-15T14:16:48,735 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:48,735 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:B 2024-11-15T14:16:48,751 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#C#compaction#437 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:48,751 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/b837c0c2b6f5452e98c63d148bd0fede is 50, key is test_row_0/C:col10/1731680207128/Put/seqid=0 2024-11-15T14:16:48,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742335_1511 (size=12915) 2024-11-15T14:16:48,791 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/b837c0c2b6f5452e98c63d148bd0fede as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b837c0c2b6f5452e98c63d148bd0fede 2024-11-15T14:16:48,800 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/C of f24e805c31c506536abbd73db35b0c0f into b837c0c2b6f5452e98c63d148bd0fede(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:48,800 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:48,800 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/C, priority=11, startTime=1731680208183; duration=0sec 2024-11-15T14:16:48,800 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:48,800 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:C 2024-11-15T14:16:48,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680268892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680268892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680268893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680268893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680268912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:48,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:48,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/5fc0904288244a618bc7d1fd2e4e61b8 2024-11-15T14:16:48,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/c52eb27be3954aabbd0c7f200b39f5fd is 50, key is test_row_0/B:col10/1731680208384/Put/seqid=0 2024-11-15T14:16:49,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742336_1512 (size=12301) 2024-11-15T14:16:49,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:49,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680269212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:49,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680269212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680269212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:49,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680269212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:49,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680269238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T14:16:49,283 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:49,284 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-15T14:16:49,286 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T14:16:49,287 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:49,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:49,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T14:16:49,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/c52eb27be3954aabbd0c7f200b39f5fd 2024-11-15T14:16:49,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T14:16:49,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:49,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:49,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:49,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:49,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:49,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/e9deb780fb804fbba06a02519988da35 is 50, key is test_row_0/C:col10/1731680208384/Put/seqid=0 2024-11-15T14:16:49,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742337_1513 (size=12301) 2024-11-15T14:16:49,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T14:16:49,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:49,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:49,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:49,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:49,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T14:16:49,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680269732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680269732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680269733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:49,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T14:16:49,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:49,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:49,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:49,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:49,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:49,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680269742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:49,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680269752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:49,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/e9deb780fb804fbba06a02519988da35 2024-11-15T14:16:49,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/5fc0904288244a618bc7d1fd2e4e61b8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/5fc0904288244a618bc7d1fd2e4e61b8 2024-11-15T14:16:49,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/5fc0904288244a618bc7d1fd2e4e61b8, entries=200, sequenceid=308, filesize=14.4 K 2024-11-15T14:16:49,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/c52eb27be3954aabbd0c7f200b39f5fd as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c52eb27be3954aabbd0c7f200b39f5fd 2024-11-15T14:16:49,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T14:16:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:49,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:49,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c52eb27be3954aabbd0c7f200b39f5fd, entries=150, sequenceid=308, filesize=12.0 K 2024-11-15T14:16:49,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/e9deb780fb804fbba06a02519988da35 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e9deb780fb804fbba06a02519988da35 2024-11-15T14:16:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T14:16:49,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e9deb780fb804fbba06a02519988da35, entries=150, sequenceid=308, filesize=12.0 K 2024-11-15T14:16:49,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f24e805c31c506536abbd73db35b0c0f in 1422ms, sequenceid=308, compaction requested=true 2024-11-15T14:16:49,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:49,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:49,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:49,923 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:49,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:49,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:49,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:49,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:16:49,925 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:49,926 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:49,926 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/A is initiating minor compaction (all files) 2024-11-15T14:16:49,926 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/A in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:49,926 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/5efe79e454e64e8cb8dcaa84d300251c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/79501c102ee34a5998a8c17cdb668085, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/5fc0904288244a618bc7d1fd2e4e61b8] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=38.9 K 2024-11-15T14:16:49,926 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5efe79e454e64e8cb8dcaa84d300251c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731680205837 2024-11-15T14:16:49,927 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:49,927 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/B is initiating minor compaction (all files) 2024-11-15T14:16:49,927 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/B in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:49,927 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/48a25c2b80594e7bb2ef98c48e3e6d1e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/42cfc0c7ec4d462eba58e9f7cd0622f0, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c52eb27be3954aabbd0c7f200b39f5fd] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=36.5 K 2024-11-15T14:16:49,927 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 79501c102ee34a5998a8c17cdb668085, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731680207124 2024-11-15T14:16:49,927 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a25c2b80594e7bb2ef98c48e3e6d1e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731680205837 2024-11-15T14:16:49,927 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fc0904288244a618bc7d1fd2e4e61b8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731680208346 2024-11-15T14:16:49,928 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42cfc0c7ec4d462eba58e9f7cd0622f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731680207124 2024-11-15T14:16:49,929 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c52eb27be3954aabbd0c7f200b39f5fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731680208368 2024-11-15T14:16:49,941 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#A#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:49,942 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/af3255386fe94a71865a772302132473 is 50, key is test_row_0/A:col10/1731680208384/Put/seqid=0 2024-11-15T14:16:49,944 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:49,945 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/3c6c90347750420c8b3980a85e4b69ad is 50, key is test_row_0/B:col10/1731680208384/Put/seqid=0 2024-11-15T14:16:50,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742338_1514 (size=12847) 2024-11-15T14:16:50,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742339_1515 (size=12847) 2024-11-15T14:16:50,041 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/3c6c90347750420c8b3980a85e4b69ad as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/3c6c90347750420c8b3980a85e4b69ad 2024-11-15T14:16:50,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T14:16:50,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:50,054 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T14:16:50,055 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/B of f24e805c31c506536abbd73db35b0c0f into 3c6c90347750420c8b3980a85e4b69ad(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:50,055 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:50,055 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/B, priority=13, startTime=1731680209923; duration=0sec 2024-11-15T14:16:50,055 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:50,055 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:B 2024-11-15T14:16:50,055 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T14:16:50,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:50,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:50,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:50,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:50,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:50,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:50,063 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T14:16:50,063 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T14:16:50,063 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. because compaction request was cancelled 2024-11-15T14:16:50,063 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:C 2024-11-15T14:16:50,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/ef0038b2d79745b0872f9171e031e8dd is 50, key is test_row_0/A:col10/1731680208567/Put/seqid=0 2024-11-15T14:16:50,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742340_1516 (size=12301) 2024-11-15T14:16:50,412 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/af3255386fe94a71865a772302132473 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/af3255386fe94a71865a772302132473 2024-11-15T14:16:50,418 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/A of f24e805c31c506536abbd73db35b0c0f into af3255386fe94a71865a772302132473(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:50,419 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:50,420 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/A, priority=13, startTime=1731680209923; duration=0sec 2024-11-15T14:16:50,420 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:50,420 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:A 2024-11-15T14:16:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T14:16:50,509 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/ef0038b2d79745b0872f9171e031e8dd 2024-11-15T14:16:50,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/b0bb587ea9d140949442e7f240401e9d is 50, key is test_row_0/B:col10/1731680208567/Put/seqid=0 2024-11-15T14:16:50,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742341_1517 (size=12301) 2024-11-15T14:16:50,608 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/b0bb587ea9d140949442e7f240401e9d 2024-11-15T14:16:50,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/487dfe36a57d4acb8f72cc3db8bb32be is 50, key is test_row_0/C:col10/1731680208567/Put/seqid=0 2024-11-15T14:16:50,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742342_1518 (size=12301) 2024-11-15T14:16:50,679 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/487dfe36a57d4acb8f72cc3db8bb32be 2024-11-15T14:16:50,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/ef0038b2d79745b0872f9171e031e8dd as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ef0038b2d79745b0872f9171e031e8dd 2024-11-15T14:16:50,692 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ef0038b2d79745b0872f9171e031e8dd, entries=150, sequenceid=328, filesize=12.0 K 2024-11-15T14:16:50,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/b0bb587ea9d140949442e7f240401e9d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/b0bb587ea9d140949442e7f240401e9d 2024-11-15T14:16:50,697 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/b0bb587ea9d140949442e7f240401e9d, entries=150, sequenceid=328, filesize=12.0 K 2024-11-15T14:16:50,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/487dfe36a57d4acb8f72cc3db8bb32be as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/487dfe36a57d4acb8f72cc3db8bb32be 2024-11-15T14:16:50,709 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/487dfe36a57d4acb8f72cc3db8bb32be, entries=150, sequenceid=328, filesize=12.0 K 2024-11-15T14:16:50,710 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for f24e805c31c506536abbd73db35b0c0f in 656ms, sequenceid=328, compaction requested=true 2024-11-15T14:16:50,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:50,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:50,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-15T14:16:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-15T14:16:50,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-15T14:16:50,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4280 sec 2024-11-15T14:16:50,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.4380 sec 2024-11-15T14:16:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:50,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T14:16:50,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:50,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:50,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:50,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:50,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:50,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:50,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/79f6e61074e242658b96cd6b20a63375 is 50, key is test_row_0/A:col10/1731680210775/Put/seqid=0 2024-11-15T14:16:50,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742343_1519 (size=14741) 2024-11-15T14:16:50,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/79f6e61074e242658b96cd6b20a63375 2024-11-15T14:16:50,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/7c17779a2399434caccfc157da3966f7 is 50, key is test_row_0/B:col10/1731680210775/Put/seqid=0 2024-11-15T14:16:50,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680270824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:50,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731680210802, max timestamp=1731680210833Num uniques 859; 2024-11-15T14:16:50,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731680210802, max timestamp=1731680210833Num uniques 859; 2024-11-15T14:16:50,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731680210802, max timestamp=1731680210833Num uniques 859; 2024-11-15T14:16:50,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T14:16:50,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T14:16:50,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T14:16:50,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:50,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:50,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:50,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742344_1520 (size=12301) 2024-11-15T14:16:50,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/7c17779a2399434caccfc157da3966f7 2024-11-15T14:16:50,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/9cbbbd3aed4143db937f8cf4566ca6dd is 50, key is test_row_0/C:col10/1731680210775/Put/seqid=0 2024-11-15T14:16:50,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742345_1521 (size=12301) 2024-11-15T14:16:50,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/9cbbbd3aed4143db937f8cf4566ca6dd 2024-11-15T14:16:50,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680270874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:50,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680270876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:50,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680270876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:50,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/79f6e61074e242658b96cd6b20a63375 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/79f6e61074e242658b96cd6b20a63375 2024-11-15T14:16:50,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/79f6e61074e242658b96cd6b20a63375, entries=200, sequenceid=340, filesize=14.4 K 2024-11-15T14:16:50,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/7c17779a2399434caccfc157da3966f7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7c17779a2399434caccfc157da3966f7 2024-11-15T14:16:50,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680270884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:50,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7c17779a2399434caccfc157da3966f7, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T14:16:50,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/9cbbbd3aed4143db937f8cf4566ca6dd as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/9cbbbd3aed4143db937f8cf4566ca6dd 2024-11-15T14:16:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:50,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/9cbbbd3aed4143db937f8cf4566ca6dd, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T14:16:50,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f24e805c31c506536abbd73db35b0c0f in 139ms, sequenceid=340, compaction requested=true 2024-11-15T14:16:50,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:50,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:50,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:50,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:50,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:16:50,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:50,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T14:16:50,916 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T14:16:50,916 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:50,917 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:50,917 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/A is initiating minor compaction (all files) 2024-11-15T14:16:50,917 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/A in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:50,918 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/af3255386fe94a71865a772302132473, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ef0038b2d79745b0872f9171e031e8dd, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/79f6e61074e242658b96cd6b20a63375] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=39.0 K 2024-11-15T14:16:50,920 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T14:16:50,920 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting af3255386fe94a71865a772302132473, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731680208368 2024-11-15T14:16:50,921 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/C is initiating minor compaction (all files) 2024-11-15T14:16:50,921 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/C in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:50,921 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b837c0c2b6f5452e98c63d148bd0fede, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e9deb780fb804fbba06a02519988da35, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/487dfe36a57d4acb8f72cc3db8bb32be, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/9cbbbd3aed4143db937f8cf4566ca6dd] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=48.7 K 2024-11-15T14:16:50,921 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting ef0038b2d79745b0872f9171e031e8dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731680208550 2024-11-15T14:16:50,921 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 79f6e61074e242658b96cd6b20a63375, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731680210768 2024-11-15T14:16:50,921 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b837c0c2b6f5452e98c63d148bd0fede, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731680207124 2024-11-15T14:16:50,922 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9deb780fb804fbba06a02519988da35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731680208368 2024-11-15T14:16:50,922 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 487dfe36a57d4acb8f72cc3db8bb32be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731680208550 2024-11-15T14:16:50,922 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cbbbd3aed4143db937f8cf4566ca6dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731680210768 2024-11-15T14:16:50,937 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#A#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:50,937 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/b661aa6c37464fba89f0b8529e12e691 is 50, key is test_row_0/A:col10/1731680210775/Put/seqid=0 2024-11-15T14:16:50,940 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#C#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:50,941 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/a943d3160e044c7990055fbc8b8fa7c2 is 50, key is test_row_0/C:col10/1731680210775/Put/seqid=0 2024-11-15T14:16:50,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-15T14:16:50,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:50,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:50,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:50,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:50,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:50,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:50,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742346_1522 (size=12949) 2024-11-15T14:16:50,969 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/b661aa6c37464fba89f0b8529e12e691 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/b661aa6c37464fba89f0b8529e12e691 2024-11-15T14:16:50,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/c8e3587c524840789b9d71d63648dd6e is 50, key is test_row_0/A:col10/1731680210879/Put/seqid=0 2024-11-15T14:16:50,977 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/A of f24e805c31c506536abbd73db35b0c0f into b661aa6c37464fba89f0b8529e12e691(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:50,977 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:50,977 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/A, priority=13, startTime=1731680210915; duration=0sec 2024-11-15T14:16:50,977 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:50,977 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:A 2024-11-15T14:16:50,977 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:50,978 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:50,978 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/B is initiating minor compaction (all files) 2024-11-15T14:16:50,978 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/B in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:50,978 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/3c6c90347750420c8b3980a85e4b69ad, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/b0bb587ea9d140949442e7f240401e9d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7c17779a2399434caccfc157da3966f7] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=36.6 K 2024-11-15T14:16:50,978 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c6c90347750420c8b3980a85e4b69ad, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731680208368 2024-11-15T14:16:50,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742347_1523 (size=13051) 2024-11-15T14:16:50,979 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting b0bb587ea9d140949442e7f240401e9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731680208550 2024-11-15T14:16:50,979 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c17779a2399434caccfc157da3966f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731680210768 2024-11-15T14:16:50,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742348_1524 (size=14741) 2024-11-15T14:16:50,993 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#B#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:50,993 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/7c9dba22bc8541558609e6c757dfb848 is 50, key is test_row_0/B:col10/1731680210775/Put/seqid=0 2024-11-15T14:16:51,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742349_1525 (size=12949) 2024-11-15T14:16:51,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680271002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680271014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680271017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680271018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680271023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680271132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680271141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680271141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680271222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680271242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680271353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680271361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680271361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,383 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/a943d3160e044c7990055fbc8b8fa7c2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a943d3160e044c7990055fbc8b8fa7c2 2024-11-15T14:16:51,387 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/C of f24e805c31c506536abbd73db35b0c0f into a943d3160e044c7990055fbc8b8fa7c2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:51,387 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:51,387 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/C, priority=12, startTime=1731680210915; duration=0sec 2024-11-15T14:16:51,387 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:51,387 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:C 2024-11-15T14:16:51,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/c8e3587c524840789b9d71d63648dd6e 2024-11-15T14:16:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,409 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/7c9dba22bc8541558609e6c757dfb848 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7c9dba22bc8541558609e6c757dfb848 2024-11-15T14:16:51,414 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/B of f24e805c31c506536abbd73db35b0c0f into 7c9dba22bc8541558609e6c757dfb848(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:51,414 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:51,414 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/B, priority=13, startTime=1731680210915; duration=0sec 2024-11-15T14:16:51,414 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:51,414 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:B 2024-11-15T14:16:51,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/86534d01638546aa9a175f5555ca8c1a is 50, key is test_row_0/B:col10/1731680210879/Put/seqid=0 2024-11-15T14:16:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T14:16:51,432 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:51,433 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:16:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-15T14:16:51,435 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:16:51,436 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:16:51,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:16:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T14:16:51,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742350_1526 (size=12301) 2024-11-15T14:16:51,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/86534d01638546aa9a175f5555ca8c1a 2024-11-15T14:16:51,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/a5a79865e1cc4c239d6e4fa26b770769 is 50, key is test_row_0/C:col10/1731680210879/Put/seqid=0 2024-11-15T14:16:51,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742351_1527 (size=12301) 2024-11-15T14:16:51,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/a5a79865e1cc4c239d6e4fa26b770769 2024-11-15T14:16:51,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/c8e3587c524840789b9d71d63648dd6e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/c8e3587c524840789b9d71d63648dd6e 2024-11-15T14:16:51,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/c8e3587c524840789b9d71d63648dd6e, entries=200, sequenceid=375, filesize=14.4 K 2024-11-15T14:16:51,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/86534d01638546aa9a175f5555ca8c1a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/86534d01638546aa9a175f5555ca8c1a 2024-11-15T14:16:51,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/86534d01638546aa9a175f5555ca8c1a, entries=150, sequenceid=375, filesize=12.0 K 2024-11-15T14:16:51,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/a5a79865e1cc4c239d6e4fa26b770769 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a5a79865e1cc4c239d6e4fa26b770769 2024-11-15T14:16:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T14:16:51,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680271544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a5a79865e1cc4c239d6e4fa26b770769, entries=150, sequenceid=375, filesize=12.0 K 2024-11-15T14:16:51,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for f24e805c31c506536abbd73db35b0c0f in 613ms, sequenceid=375, compaction requested=false 2024-11-15T14:16:51,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:51,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T14:16:51,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:51,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:51,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:51,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:51,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:51,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:51,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/a4efc02ea828425fbe39512348aeb356 is 50, key is test_row_1/A:col10/1731680211569/Put/seqid=0 2024-11-15T14:16:51,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:51,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:51,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:51,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:51,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:51,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:51,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742352_1528 (size=12297) 2024-11-15T14:16:51,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/a4efc02ea828425fbe39512348aeb356 2024-11-15T14:16:51,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/1c28ec807dd24293afa3dceede0da3b9 is 50, key is test_row_1/B:col10/1731680211569/Put/seqid=0 2024-11-15T14:16:51,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742353_1529 (size=9857) 2024-11-15T14:16:51,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680271705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731680211697, max timestamp=1731680211719Num uniques 859; 2024-11-15T14:16:51,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T14:16:51,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731680211703, max timestamp=1731680211719Num uniques 859; 2024-11-15T14:16:51,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T14:16:51,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731680211697, max timestamp=1731680211719Num uniques 859; 2024-11-15T14:16:51,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T14:16:51,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:51,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:51,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:51,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:51,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:51,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T14:16:51,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680271801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680271803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680271805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680271832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:51,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:51,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:51,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:51,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:51,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:51,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680271921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680271921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680271921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:52,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:52,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680272052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680272052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T14:16:52,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/1c28ec807dd24293afa3dceede0da3b9 2024-11-15T14:16:52,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/55fe49fa604049948c4f81e45e1d19ab is 50, key is test_row_1/C:col10/1731680211569/Put/seqid=0 2024-11-15T14:16:52,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742354_1530 (size=9857) 2024-11-15T14:16:52,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/55fe49fa604049948c4f81e45e1d19ab 2024-11-15T14:16:52,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/a4efc02ea828425fbe39512348aeb356 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a4efc02ea828425fbe39512348aeb356 2024-11-15T14:16:52,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a4efc02ea828425fbe39512348aeb356, entries=150, sequenceid=389, filesize=12.0 K 2024-11-15T14:16:52,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/1c28ec807dd24293afa3dceede0da3b9 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/1c28ec807dd24293afa3dceede0da3b9 2024-11-15T14:16:52,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/1c28ec807dd24293afa3dceede0da3b9, entries=100, sequenceid=389, filesize=9.6 K 2024-11-15T14:16:52,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/55fe49fa604049948c4f81e45e1d19ab as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/55fe49fa604049948c4f81e45e1d19ab 2024-11-15T14:16:52,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/55fe49fa604049948c4f81e45e1d19ab, entries=100, sequenceid=389, filesize=9.6 K 2024-11-15T14:16:52,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.31 KB/132415 for f24e805c31c506536abbd73db35b0c0f in 518ms, sequenceid=389, compaction requested=true 2024-11-15T14:16:52,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:52,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:52,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:52,089 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:52,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:52,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:52,089 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:52,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:52,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:52,089 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39987 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:52,089 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:52,089 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/B is initiating minor compaction (all files) 2024-11-15T14:16:52,089 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/A is initiating minor compaction (all files) 2024-11-15T14:16:52,089 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/B in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,089 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/A in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,089 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/b661aa6c37464fba89f0b8529e12e691, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/c8e3587c524840789b9d71d63648dd6e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a4efc02ea828425fbe39512348aeb356] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=39.0 K 2024-11-15T14:16:52,089 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7c9dba22bc8541558609e6c757dfb848, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/86534d01638546aa9a175f5555ca8c1a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/1c28ec807dd24293afa3dceede0da3b9] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=34.3 K 2024-11-15T14:16:52,089 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b661aa6c37464fba89f0b8529e12e691, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731680210768 2024-11-15T14:16:52,089 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c9dba22bc8541558609e6c757dfb848, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731680210768 2024-11-15T14:16:52,090 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 86534d01638546aa9a175f5555ca8c1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731680210872 2024-11-15T14:16:52,090 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8e3587c524840789b9d71d63648dd6e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731680210872 2024-11-15T14:16:52,090 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c28ec807dd24293afa3dceede0da3b9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731680211017 2024-11-15T14:16:52,090 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4efc02ea828425fbe39512348aeb356, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731680210998 2024-11-15T14:16:52,096 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#B#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:52,097 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#A#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:52,097 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/b27ef17d78ad4acdbdbda9268a31788e is 50, key is test_row_0/B:col10/1731680210879/Put/seqid=0 2024-11-15T14:16:52,097 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/582cb2f07f954c049280d4be9f25eabd is 50, key is test_row_0/A:col10/1731680210879/Put/seqid=0 2024-11-15T14:16:52,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742356_1532 (size=13051) 2024-11-15T14:16:52,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742355_1531 (size=13051) 2024-11-15T14:16:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:52,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=136.02 KB heapSize=384.46 KB 2024-11-15T14:16:52,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:52,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:52,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:52,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:52,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:52,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:52,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/46f2c578b47f423aa388b029cb2ce0f1 is 50, key is test_row_0/A:col10/1731680211800/Put/seqid=0 2024-11-15T14:16:52,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742357_1533 (size=14741) 2024-11-15T14:16:52,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680272152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680272153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680272154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680272262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680272262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680272271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680272361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680272471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680272471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680272491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,508 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/582cb2f07f954c049280d4be9f25eabd as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/582cb2f07f954c049280d4be9f25eabd 2024-11-15T14:16:52,508 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/b27ef17d78ad4acdbdbda9268a31788e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/b27ef17d78ad4acdbdbda9268a31788e 2024-11-15T14:16:52,513 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/B of f24e805c31c506536abbd73db35b0c0f into b27ef17d78ad4acdbdbda9268a31788e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:52,513 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:52,513 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/B, priority=13, startTime=1731680212089; duration=0sec 2024-11-15T14:16:52,514 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:52,514 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:B 2024-11-15T14:16:52,514 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:52,514 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/A of f24e805c31c506536abbd73db35b0c0f into 582cb2f07f954c049280d4be9f25eabd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:52,514 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:52,514 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/A, priority=13, startTime=1731680212088; duration=0sec 2024-11-15T14:16:52,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,517 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:52,517 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:A 2024-11-15T14:16:52,518 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:52,518 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/C is initiating minor compaction (all files) 2024-11-15T14:16:52,518 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/C in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,518 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a943d3160e044c7990055fbc8b8fa7c2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a5a79865e1cc4c239d6e4fa26b770769, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/55fe49fa604049948c4f81e45e1d19ab] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=34.4 K 2024-11-15T14:16:52,518 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting a943d3160e044c7990055fbc8b8fa7c2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731680210768 2024-11-15T14:16:52,523 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting a5a79865e1cc4c239d6e4fa26b770769, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731680210872 2024-11-15T14:16:52,524 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 55fe49fa604049948c4f81e45e1d19ab, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731680211017 2024-11-15T14:16:52,536 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#C#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:52,537 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/308342621e384b88b7159f5893e5106c is 50, key is test_row_0/C:col10/1731680210879/Put/seqid=0 2024-11-15T14:16:52,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742358_1534 (size=13153) 2024-11-15T14:16:52,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/46f2c578b47f423aa388b029cb2ce0f1 2024-11-15T14:16:52,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/c6de1cbed69d41ae950f1af23726dac5 is 50, key is test_row_0/B:col10/1731680211800/Put/seqid=0 2024-11-15T14:16:52,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742359_1535 (size=12301) 2024-11-15T14:16:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T14:16:52,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680272792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680272792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680272801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680272872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:52,947 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/308342621e384b88b7159f5893e5106c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/308342621e384b88b7159f5893e5106c 2024-11-15T14:16:52,950 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/C of f24e805c31c506536abbd73db35b0c0f into 308342621e384b88b7159f5893e5106c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:52,950 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:52,950 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/C, priority=13, startTime=1731680212089; duration=0sec 2024-11-15T14:16:52,950 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:52,950 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:C 2024-11-15T14:16:52,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/c6de1cbed69d41ae950f1af23726dac5 2024-11-15T14:16:52,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/b92ddbaa2a0840c491a0ffb5fe46fe73 is 50, key is test_row_0/C:col10/1731680211800/Put/seqid=0 2024-11-15T14:16:52,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742360_1536 (size=12301) 2024-11-15T14:16:52,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/b92ddbaa2a0840c491a0ffb5fe46fe73 2024-11-15T14:16:52,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/46f2c578b47f423aa388b029cb2ce0f1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/46f2c578b47f423aa388b029cb2ce0f1 2024-11-15T14:16:52,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:52,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:52,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:52,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:52,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/46f2c578b47f423aa388b029cb2ce0f1, entries=200, sequenceid=425, filesize=14.4 K 2024-11-15T14:16:52,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/c6de1cbed69d41ae950f1af23726dac5 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c6de1cbed69d41ae950f1af23726dac5 2024-11-15T14:16:52,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c6de1cbed69d41ae950f1af23726dac5, entries=150, sequenceid=425, filesize=12.0 K 2024-11-15T14:16:52,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/b92ddbaa2a0840c491a0ffb5fe46fe73 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b92ddbaa2a0840c491a0ffb5fe46fe73 2024-11-15T14:16:52,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b92ddbaa2a0840c491a0ffb5fe46fe73, entries=150, sequenceid=425, filesize=12.0 K 2024-11-15T14:16:52,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.02 KB/139285, heapSize ~384.41 KB/393640, currentSize=46.96 KB/48090 for f24e805c31c506536abbd73db35b0c0f in 847ms, sequenceid=425, compaction requested=false 2024-11-15T14:16:52,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:53,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T14:16:53,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:53,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:53,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:53,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:53,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:53,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:53,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/d37d8c138c004ab88577694bf2f53859 is 50, key is test_row_0/A:col10/1731680213075/Put/seqid=0 2024-11-15T14:16:53,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742361_1537 (size=14741) 2024-11-15T14:16:53,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680213108, max timestamp=1731680213210Num uniques 859; 2024-11-15T14:16:53,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680213108, max timestamp=1731680213210Num uniques 859; 2024-11-15T14:16:53,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:16:53,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:16:53,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680213108, max timestamp=1731680213210Num uniques 859; 2024-11-15T14:16:53,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:16:53,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680273311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:53,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680273311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680273314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680273323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:53,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680273431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:53,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:53,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:53,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680273442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:53,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/d37d8c138c004ab88577694bf2f53859 2024-11-15T14:16:53,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/a0c724b20d284102a6b9b3739c621e12 is 50, key is test_row_0/B:col10/1731680213075/Put/seqid=0 2024-11-15T14:16:53,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742362_1538 (size=12301) 2024-11-15T14:16:53,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/a0c724b20d284102a6b9b3739c621e12 2024-11-15T14:16:53,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/60eb4d36971949ab9eff283aa4c8d666 is 50, key is test_row_0/C:col10/1731680213075/Put/seqid=0 2024-11-15T14:16:53,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742363_1539 (size=12301) 2024-11-15T14:16:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T14:16:53,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:53,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:53,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680273642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:53,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:53,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680273662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:53,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:53,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:53,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:53,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56752 deadline: 1731680273892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:53,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:53,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,897 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in f24e805c31c506536abbd73db35b0c0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:16:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:53,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/60eb4d36971949ab9eff283aa4c8d666 2024-11-15T14:16:53,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/d37d8c138c004ab88577694bf2f53859 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d37d8c138c004ab88577694bf2f53859 2024-11-15T14:16:53,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d37d8c138c004ab88577694bf2f53859, entries=200, sequenceid=439, filesize=14.4 K 2024-11-15T14:16:53,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/a0c724b20d284102a6b9b3739c621e12 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/a0c724b20d284102a6b9b3739c621e12 2024-11-15T14:16:53,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/a0c724b20d284102a6b9b3739c621e12, entries=150, sequenceid=439, filesize=12.0 K 2024-11-15T14:16:53,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/60eb4d36971949ab9eff283aa4c8d666 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/60eb4d36971949ab9eff283aa4c8d666 2024-11-15T14:16:53,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/60eb4d36971949ab9eff283aa4c8d666, entries=150, sequenceid=439, filesize=12.0 K 2024-11-15T14:16:53,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f24e805c31c506536abbd73db35b0c0f in 845ms, sequenceid=439, compaction requested=true 2024-11-15T14:16:53,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:53,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:16:53,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:53,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:16:53,921 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:53,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:53,921 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:53,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f24e805c31c506536abbd73db35b0c0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:16:53,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:53,922 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:53,922 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:53,922 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/A is initiating minor compaction (all files) 2024-11-15T14:16:53,922 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/B is initiating minor compaction (all files) 2024-11-15T14:16:53,922 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/A in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,922 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/582cb2f07f954c049280d4be9f25eabd, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/46f2c578b47f423aa388b029cb2ce0f1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d37d8c138c004ab88577694bf2f53859] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=41.5 K 2024-11-15T14:16:53,922 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/B in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,922 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/b27ef17d78ad4acdbdbda9268a31788e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c6de1cbed69d41ae950f1af23726dac5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/a0c724b20d284102a6b9b3739c621e12] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=36.8 K 2024-11-15T14:16:53,922 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 582cb2f07f954c049280d4be9f25eabd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731680210879 2024-11-15T14:16:53,922 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b27ef17d78ad4acdbdbda9268a31788e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731680210879 2024-11-15T14:16:53,923 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6de1cbed69d41ae950f1af23726dac5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731680211800 2024-11-15T14:16:53,923 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 46f2c578b47f423aa388b029cb2ce0f1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731680211765 2024-11-15T14:16:53,923 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0c724b20d284102a6b9b3739c621e12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731680212152 2024-11-15T14:16:53,923 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting d37d8c138c004ab88577694bf2f53859, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731680212151 2024-11-15T14:16:53,941 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#B#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:53,941 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#A#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:53,941 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/21d7a246f35d4e1f80e11aa82e41a4fa is 50, key is test_row_0/B:col10/1731680213075/Put/seqid=0 2024-11-15T14:16:53,941 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/fbbcd8e75bfb4c2192e6bccd09c792fc is 50, key is test_row_0/A:col10/1731680213075/Put/seqid=0 2024-11-15T14:16:53,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742364_1540 (size=13153) 2024-11-15T14:16:53,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742365_1541 (size=13153) 2024-11-15T14:16:53,954 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/21d7a246f35d4e1f80e11aa82e41a4fa as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/21d7a246f35d4e1f80e11aa82e41a4fa 2024-11-15T14:16:53,958 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/fbbcd8e75bfb4c2192e6bccd09c792fc as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/fbbcd8e75bfb4c2192e6bccd09c792fc 2024-11-15T14:16:53,962 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/A of f24e805c31c506536abbd73db35b0c0f into fbbcd8e75bfb4c2192e6bccd09c792fc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:53,962 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:53,962 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/A, priority=13, startTime=1731680213921; duration=0sec 2024-11-15T14:16:53,962 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:16:53,962 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:A 2024-11-15T14:16:53,962 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:16:53,964 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:16:53,964 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): f24e805c31c506536abbd73db35b0c0f/C is initiating minor compaction (all files) 2024-11-15T14:16:53,964 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/B of f24e805c31c506536abbd73db35b0c0f into 21d7a246f35d4e1f80e11aa82e41a4fa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:53,964 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:53,964 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f24e805c31c506536abbd73db35b0c0f/C in TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:53,964 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/B, priority=13, startTime=1731680213921; duration=0sec 2024-11-15T14:16:53,964 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/308342621e384b88b7159f5893e5106c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b92ddbaa2a0840c491a0ffb5fe46fe73, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/60eb4d36971949ab9eff283aa4c8d666] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp, totalSize=36.9 K 2024-11-15T14:16:53,964 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:53,964 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:B 2024-11-15T14:16:53,964 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 308342621e384b88b7159f5893e5106c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731680210879 2024-11-15T14:16:53,965 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting b92ddbaa2a0840c491a0ffb5fe46fe73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731680211800 2024-11-15T14:16:53,965 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 60eb4d36971949ab9eff283aa4c8d666, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731680212152 2024-11-15T14:16:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:53,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-15T14:16:53,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:53,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:53,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:53,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:53,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:53,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:16:53,971 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f24e805c31c506536abbd73db35b0c0f#C#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:16:53,972 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/7e1b445e192048ef8695f6b50019b733 is 50, key is test_row_0/C:col10/1731680213075/Put/seqid=0 2024-11-15T14:16:53,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/7468cef0033a4d10a87cd4f83f0eeee3 is 50, key is test_row_0/A:col10/1731680213308/Put/seqid=0 2024-11-15T14:16:53,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742366_1542 (size=14741) 2024-11-15T14:16:53,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742367_1543 (size=13255) 2024-11-15T14:16:53,986 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/7e1b445e192048ef8695f6b50019b733 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/7e1b445e192048ef8695f6b50019b733 2024-11-15T14:16:53,993 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f24e805c31c506536abbd73db35b0c0f/C of f24e805c31c506536abbd73db35b0c0f into 7e1b445e192048ef8695f6b50019b733(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:16:53,993 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:53,993 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., storeName=f24e805c31c506536abbd73db35b0c0f/C, priority=13, startTime=1731680213921; duration=0sec 2024-11-15T14:16:53,993 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:16:53,993 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f24e805c31c506536abbd73db35b0c0f:C 2024-11-15T14:16:54,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680274013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:54,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680274013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:54,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:54,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:54,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680274122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680274122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:54,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:54,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:54,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56748 deadline: 1731680274332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:54,333 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in f24e805c31c506536abbd73db35b0c0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:16:54,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:54,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1731680274342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:54,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680274342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:54,344 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in f24e805c31c506536abbd73db35b0c0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:16:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680274342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:54,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:54,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:54,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/7468cef0033a4d10a87cd4f83f0eeee3 2024-11-15T14:16:54,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/7d570564694445d59128b9eb1e204a99 is 50, key is test_row_0/B:col10/1731680213308/Put/seqid=0 2024-11-15T14:16:54,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742368_1544 (size=12301) 2024-11-15T14:16:54,504 INFO [Thread-2002 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2002. 2024-11-15T14:16:54,504 DEBUG [Thread-2002 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:54,504 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:54,505 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:54,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:54,505 INFO [Thread-2000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2000. 2024-11-15T14:16:54,505 DEBUG [Thread-2000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:54,505 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:54,505 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:54,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:54,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:54,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:54,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,511 INFO [Thread-2004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2004. 2024-11-15T14:16:54,511 DEBUG [Thread-2004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:54,511 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:54,511 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:54,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:54,515 INFO [Thread-2006 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2006. 2024-11-15T14:16:54,515 DEBUG [Thread-2006 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:54,515 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:54,515 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:54,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:54,517 INFO [Thread-2008 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2008. 2024-11-15T14:16:54,517 DEBUG [Thread-2008 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:54,517 DEBUG [Thread-2008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:54,517 DEBUG [Thread-2008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:54,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680274653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680274653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:54,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:54,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:54,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/7d570564694445d59128b9eb1e204a99 2024-11-15T14:16:54,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/1aed7f6f87f14b5092bfa5ec6b79fd41 is 50, key is test_row_0/C:col10/1731680213308/Put/seqid=0 2024-11-15T14:16:54,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742369_1545 (size=12301) 2024-11-15T14:16:54,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:54,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:54,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:54,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:55,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:55,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:55,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:16:55,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:55,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:16:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1731680275162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56780 deadline: 1731680275162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 2024-11-15T14:16:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 , the old value is region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:16:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., hostname=26754e923c4a,40177,1731680077530, seqNum=2 because the exception is null or not the one we care about 2024-11-15T14:16:55,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/1aed7f6f87f14b5092bfa5ec6b79fd41 2024-11-15T14:16:55,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/7468cef0033a4d10a87cd4f83f0eeee3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/7468cef0033a4d10a87cd4f83f0eeee3 2024-11-15T14:16:55,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/7468cef0033a4d10a87cd4f83f0eeee3, entries=200, sequenceid=477, filesize=14.4 K 2024-11-15T14:16:55,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/7d570564694445d59128b9eb1e204a99 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7d570564694445d59128b9eb1e204a99 2024-11-15T14:16:55,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7d570564694445d59128b9eb1e204a99, entries=150, sequenceid=477, filesize=12.0 K 2024-11-15T14:16:55,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/1aed7f6f87f14b5092bfa5ec6b79fd41 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/1aed7f6f87f14b5092bfa5ec6b79fd41 2024-11-15T14:16:55,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/1aed7f6f87f14b5092bfa5ec6b79fd41, entries=150, sequenceid=477, filesize=12.0 K 2024-11-15T14:16:55,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for f24e805c31c506536abbd73db35b0c0f in 1257ms, sequenceid=477, compaction requested=false 2024-11-15T14:16:55,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:55,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T14:16:55,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:55,282 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T14:16:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:55,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/d8044720e8124d5d820539f5f8b7af23 is 50, key is test_row_0/A:col10/1731680214000/Put/seqid=0 2024-11-15T14:16:55,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742370_1546 (size=12301) 2024-11-15T14:16:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T14:16:55,693 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/d8044720e8124d5d820539f5f8b7af23 2024-11-15T14:16:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/e70fb6adf09a4700b6c3ab78329607f2 is 50, key is test_row_0/B:col10/1731680214000/Put/seqid=0 2024-11-15T14:16:55,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742371_1547 (size=12301) 2024-11-15T14:16:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:55,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. as already flushing 2024-11-15T14:16:55,906 INFO [Thread-1997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1997. 2024-11-15T14:16:55,906 DEBUG [Thread-1997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:55,906 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:55,906 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:55,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:56,110 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/e70fb6adf09a4700b6c3ab78329607f2 2024-11-15T14:16:56,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/31926f4c6a9c4026a55d4151b5f2de86 is 50, key is test_row_0/C:col10/1731680214000/Put/seqid=0 2024-11-15T14:16:56,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742372_1548 (size=12301) 2024-11-15T14:16:56,172 INFO [Thread-1989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1989. 2024-11-15T14:16:56,173 DEBUG [Thread-1989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:56,173 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:56,173 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:56,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:56,183 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-11-15T14:16:56,183 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:56,183 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:56,183 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:56,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:56,343 INFO [Thread-1995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1995. 2024-11-15T14:16:56,343 DEBUG [Thread-1995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:56,343 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:56,343 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:56,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:56,364 INFO [Thread-1991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1991. 2024-11-15T14:16:56,364 DEBUG [Thread-1991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:16:56,364 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:56,364 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:56,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:56,527 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/31926f4c6a9c4026a55d4151b5f2de86 2024-11-15T14:16:56,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/d8044720e8124d5d820539f5f8b7af23 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d8044720e8124d5d820539f5f8b7af23 2024-11-15T14:16:56,536 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d8044720e8124d5d820539f5f8b7af23, entries=150, sequenceid=489, filesize=12.0 K 2024-11-15T14:16:56,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/e70fb6adf09a4700b6c3ab78329607f2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/e70fb6adf09a4700b6c3ab78329607f2 2024-11-15T14:16:56,539 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/e70fb6adf09a4700b6c3ab78329607f2, entries=150, sequenceid=489, filesize=12.0 K 2024-11-15T14:16:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/31926f4c6a9c4026a55d4151b5f2de86 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/31926f4c6a9c4026a55d4151b5f2de86 2024-11-15T14:16:56,542 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/31926f4c6a9c4026a55d4151b5f2de86, entries=150, sequenceid=489, filesize=12.0 K 2024-11-15T14:16:56,543 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=33.54 KB/34350 for f24e805c31c506536abbd73db35b0c0f in 1261ms, sequenceid=489, compaction requested=true 2024-11-15T14:16:56,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for f24e805c31c506536abbd73db35b0c0f: 2024-11-15T14:16:56,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:56,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-15T14:16:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-15T14:16:56,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-15T14:16:56,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.1090 sec 2024-11-15T14:16:56,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 5.1140 sec 2024-11-15T14:16:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T14:16:59,613 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 840 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2499 rows 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 842 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2526 rows 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 833 2024-11-15T14:16:59,613 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2499 rows 2024-11-15T14:16:59,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 836 2024-11-15T14:16:59,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2506 rows 2024-11-15T14:16:59,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 834 2024-11-15T14:16:59,614 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2485 rows 2024-11-15T14:16:59,614 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T14:16:59,614 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T14:16:59,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:59,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:16:59,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:16:59,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T14:16:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T14:16:59,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T14:16:59,622 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680219622"}]},"ts":"1731680219622"} 2024-11-15T14:16:59,624 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T14:16:59,624 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T14:16:59,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T14:16:59,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f24e805c31c506536abbd73db35b0c0f, UNASSIGN}] 2024-11-15T14:16:59,628 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f24e805c31c506536abbd73db35b0c0f, UNASSIGN 2024-11-15T14:16:59,629 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=f24e805c31c506536abbd73db35b0c0f, regionState=CLOSING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:16:59,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f24e805c31c506536abbd73db35b0c0f, UNASSIGN because future has completed 2024-11-15T14:16:59,631 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T14:16:59,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:16:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T14:16:59,784 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:16:59,785 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T14:16:59,785 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing f24e805c31c506536abbd73db35b0c0f, disabling compactions & flushes 2024-11-15T14:16:59,785 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:59,785 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:59,785 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. after waiting 0 ms 2024-11-15T14:16:59,785 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:16:59,785 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing f24e805c31c506536abbd73db35b0c0f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T14:16:59,786 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=A 2024-11-15T14:16:59,786 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:59,786 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=B 2024-11-15T14:16:59,786 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:59,786 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f24e805c31c506536abbd73db35b0c0f, store=C 2024-11-15T14:16:59,786 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:16:59,792 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/d3592fd64c7e48f1a188f045eb47c6ab is 50, key is test_row_1/A:col10/1731680216342/Put/seqid=0 2024-11-15T14:16:59,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742373_1549 (size=9857) 2024-11-15T14:16:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T14:17:00,198 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/d3592fd64c7e48f1a188f045eb47c6ab 2024-11-15T14:17:00,208 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/01b0e89ac5434ad7b0b71c8ce88e3afc is 50, key is test_row_1/B:col10/1731680216342/Put/seqid=0 2024-11-15T14:17:00,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742374_1550 (size=9857) 2024-11-15T14:17:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T14:17:00,613 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/01b0e89ac5434ad7b0b71c8ce88e3afc 2024-11-15T14:17:00,627 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/417ddcad532648349091b9d7e9f965cb is 50, key is test_row_1/C:col10/1731680216342/Put/seqid=0 2024-11-15T14:17:00,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742375_1551 (size=9857) 2024-11-15T14:17:00,631 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/417ddcad532648349091b9d7e9f965cb 2024-11-15T14:17:00,635 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/A/d3592fd64c7e48f1a188f045eb47c6ab as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d3592fd64c7e48f1a188f045eb47c6ab 2024-11-15T14:17:00,638 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d3592fd64c7e48f1a188f045eb47c6ab, entries=100, sequenceid=497, filesize=9.6 K 2024-11-15T14:17:00,639 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/B/01b0e89ac5434ad7b0b71c8ce88e3afc as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/01b0e89ac5434ad7b0b71c8ce88e3afc 2024-11-15T14:17:00,642 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/01b0e89ac5434ad7b0b71c8ce88e3afc, entries=100, sequenceid=497, filesize=9.6 K 2024-11-15T14:17:00,643 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/.tmp/C/417ddcad532648349091b9d7e9f965cb as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/417ddcad532648349091b9d7e9f965cb 2024-11-15T14:17:00,646 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/417ddcad532648349091b9d7e9f965cb, entries=100, sequenceid=497, filesize=9.6 K 2024-11-15T14:17:00,647 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f24e805c31c506536abbd73db35b0c0f in 862ms, sequenceid=497, compaction requested=true 2024-11-15T14:17:00,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/451ef5dff3e640b4820b27e9fe98bc5e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/1c5ce7c2f606494eb7bb936f15689293, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ddc62289047d407b9a0780370f7129c4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/1b0039cf39b34d3aafc6f98dd4871372, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/3878643e6eeb4215bc9887dd9edaf850, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/21f28a04b2b640a2bcaf4ba89d67c3be, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/852a9c7841834e9d8e2054e4217da5f6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/60cc142a96934dcda7dd6a3a91de5c4e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/590357b28ecc46da9e1dc79eaec66adc, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/eb3a5731780e4f3f84c0e2a10b18e0a6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/83a4f11b2b484e6fb2fe1c5b9ea47a26, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/0fdd984b4d63457494b7844a488c68e3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/5efe79e454e64e8cb8dcaa84d300251c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/f72e102cc0544982a6a1fc4ad52e0c5f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/79501c102ee34a5998a8c17cdb668085, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/5fc0904288244a618bc7d1fd2e4e61b8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/af3255386fe94a71865a772302132473, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ef0038b2d79745b0872f9171e031e8dd, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/79f6e61074e242658b96cd6b20a63375, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/b661aa6c37464fba89f0b8529e12e691, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/c8e3587c524840789b9d71d63648dd6e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/582cb2f07f954c049280d4be9f25eabd, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a4efc02ea828425fbe39512348aeb356, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/46f2c578b47f423aa388b029cb2ce0f1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d37d8c138c004ab88577694bf2f53859] to archive 2024-11-15T14:17:00,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:17:00,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/451ef5dff3e640b4820b27e9fe98bc5e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/451ef5dff3e640b4820b27e9fe98bc5e 2024-11-15T14:17:00,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/1c5ce7c2f606494eb7bb936f15689293 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/1c5ce7c2f606494eb7bb936f15689293 2024-11-15T14:17:00,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ddc62289047d407b9a0780370f7129c4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ddc62289047d407b9a0780370f7129c4 2024-11-15T14:17:00,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/1b0039cf39b34d3aafc6f98dd4871372 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/1b0039cf39b34d3aafc6f98dd4871372 2024-11-15T14:17:00,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/3878643e6eeb4215bc9887dd9edaf850 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/3878643e6eeb4215bc9887dd9edaf850 2024-11-15T14:17:00,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/21f28a04b2b640a2bcaf4ba89d67c3be to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/21f28a04b2b640a2bcaf4ba89d67c3be 2024-11-15T14:17:00,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/852a9c7841834e9d8e2054e4217da5f6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/852a9c7841834e9d8e2054e4217da5f6 2024-11-15T14:17:00,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/60cc142a96934dcda7dd6a3a91de5c4e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/60cc142a96934dcda7dd6a3a91de5c4e 2024-11-15T14:17:00,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/590357b28ecc46da9e1dc79eaec66adc to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/590357b28ecc46da9e1dc79eaec66adc 2024-11-15T14:17:00,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/eb3a5731780e4f3f84c0e2a10b18e0a6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/eb3a5731780e4f3f84c0e2a10b18e0a6 2024-11-15T14:17:00,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/83a4f11b2b484e6fb2fe1c5b9ea47a26 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/83a4f11b2b484e6fb2fe1c5b9ea47a26 2024-11-15T14:17:00,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/0fdd984b4d63457494b7844a488c68e3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/0fdd984b4d63457494b7844a488c68e3 2024-11-15T14:17:00,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/5efe79e454e64e8cb8dcaa84d300251c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/5efe79e454e64e8cb8dcaa84d300251c 2024-11-15T14:17:00,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/f72e102cc0544982a6a1fc4ad52e0c5f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/f72e102cc0544982a6a1fc4ad52e0c5f 2024-11-15T14:17:00,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/79501c102ee34a5998a8c17cdb668085 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/79501c102ee34a5998a8c17cdb668085 2024-11-15T14:17:00,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/5fc0904288244a618bc7d1fd2e4e61b8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/5fc0904288244a618bc7d1fd2e4e61b8 2024-11-15T14:17:00,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/af3255386fe94a71865a772302132473 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/af3255386fe94a71865a772302132473 2024-11-15T14:17:00,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ef0038b2d79745b0872f9171e031e8dd to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/ef0038b2d79745b0872f9171e031e8dd 2024-11-15T14:17:00,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/79f6e61074e242658b96cd6b20a63375 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/79f6e61074e242658b96cd6b20a63375 2024-11-15T14:17:00,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/b661aa6c37464fba89f0b8529e12e691 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/b661aa6c37464fba89f0b8529e12e691 2024-11-15T14:17:00,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/c8e3587c524840789b9d71d63648dd6e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/c8e3587c524840789b9d71d63648dd6e 2024-11-15T14:17:00,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/582cb2f07f954c049280d4be9f25eabd to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/582cb2f07f954c049280d4be9f25eabd 2024-11-15T14:17:00,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a4efc02ea828425fbe39512348aeb356 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/a4efc02ea828425fbe39512348aeb356 2024-11-15T14:17:00,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/46f2c578b47f423aa388b029cb2ce0f1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/46f2c578b47f423aa388b029cb2ce0f1 2024-11-15T14:17:00,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d37d8c138c004ab88577694bf2f53859 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d37d8c138c004ab88577694bf2f53859 2024-11-15T14:17:00,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/35d2be05199b4443af9fa3702b256228, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c37d69c50d2b4c579d89bcd1f7e8fc35, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/4b9f909f254440f89acd7aae3c7328f0, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/62e5e86d1cf34f42a52c84cba1039814, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/cc2339794c2a47568d9d275bede609f8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/2a31f172584a4fd4a3f6633dd2234fa5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/ecedb12e00e6426a9a24ca25b9cbbf61, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c1755b20b5c8487da982095bb003ca6c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/ab6d2ecc67bc4b40856428264c237087, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/78bcbe97f0174c73bb8d0e8147d7462d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/587412ba4f074251b6c05e2ca6d88166, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/44c753ae2a9b49bb80df7a3c9f6ae7be, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/48a25c2b80594e7bb2ef98c48e3e6d1e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/3fa5a1b8441b46f899950bdd410aa2dd, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/42cfc0c7ec4d462eba58e9f7cd0622f0, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/3c6c90347750420c8b3980a85e4b69ad, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c52eb27be3954aabbd0c7f200b39f5fd, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/b0bb587ea9d140949442e7f240401e9d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7c9dba22bc8541558609e6c757dfb848, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7c17779a2399434caccfc157da3966f7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/86534d01638546aa9a175f5555ca8c1a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/b27ef17d78ad4acdbdbda9268a31788e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/1c28ec807dd24293afa3dceede0da3b9, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c6de1cbed69d41ae950f1af23726dac5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/a0c724b20d284102a6b9b3739c621e12] to archive 2024-11-15T14:17:00,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:17:00,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/35d2be05199b4443af9fa3702b256228 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/35d2be05199b4443af9fa3702b256228 2024-11-15T14:17:00,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c37d69c50d2b4c579d89bcd1f7e8fc35 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c37d69c50d2b4c579d89bcd1f7e8fc35 2024-11-15T14:17:00,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/4b9f909f254440f89acd7aae3c7328f0 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/4b9f909f254440f89acd7aae3c7328f0 2024-11-15T14:17:00,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/62e5e86d1cf34f42a52c84cba1039814 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/62e5e86d1cf34f42a52c84cba1039814 2024-11-15T14:17:00,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/cc2339794c2a47568d9d275bede609f8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/cc2339794c2a47568d9d275bede609f8 2024-11-15T14:17:00,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/2a31f172584a4fd4a3f6633dd2234fa5 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/2a31f172584a4fd4a3f6633dd2234fa5 2024-11-15T14:17:00,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/ecedb12e00e6426a9a24ca25b9cbbf61 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/ecedb12e00e6426a9a24ca25b9cbbf61 2024-11-15T14:17:00,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c1755b20b5c8487da982095bb003ca6c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c1755b20b5c8487da982095bb003ca6c 2024-11-15T14:17:00,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/ab6d2ecc67bc4b40856428264c237087 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/ab6d2ecc67bc4b40856428264c237087 2024-11-15T14:17:00,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/78bcbe97f0174c73bb8d0e8147d7462d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/78bcbe97f0174c73bb8d0e8147d7462d 2024-11-15T14:17:00,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/587412ba4f074251b6c05e2ca6d88166 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/587412ba4f074251b6c05e2ca6d88166 2024-11-15T14:17:00,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/44c753ae2a9b49bb80df7a3c9f6ae7be to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/44c753ae2a9b49bb80df7a3c9f6ae7be 2024-11-15T14:17:00,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/48a25c2b80594e7bb2ef98c48e3e6d1e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/48a25c2b80594e7bb2ef98c48e3e6d1e 2024-11-15T14:17:00,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/3fa5a1b8441b46f899950bdd410aa2dd to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/3fa5a1b8441b46f899950bdd410aa2dd 2024-11-15T14:17:00,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/42cfc0c7ec4d462eba58e9f7cd0622f0 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/42cfc0c7ec4d462eba58e9f7cd0622f0 2024-11-15T14:17:00,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/3c6c90347750420c8b3980a85e4b69ad to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/3c6c90347750420c8b3980a85e4b69ad 2024-11-15T14:17:00,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c52eb27be3954aabbd0c7f200b39f5fd to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c52eb27be3954aabbd0c7f200b39f5fd 2024-11-15T14:17:00,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/b0bb587ea9d140949442e7f240401e9d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/b0bb587ea9d140949442e7f240401e9d 2024-11-15T14:17:00,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7c9dba22bc8541558609e6c757dfb848 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7c9dba22bc8541558609e6c757dfb848 2024-11-15T14:17:00,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7c17779a2399434caccfc157da3966f7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7c17779a2399434caccfc157da3966f7 2024-11-15T14:17:00,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/86534d01638546aa9a175f5555ca8c1a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/86534d01638546aa9a175f5555ca8c1a 2024-11-15T14:17:00,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/b27ef17d78ad4acdbdbda9268a31788e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/b27ef17d78ad4acdbdbda9268a31788e 2024-11-15T14:17:00,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/1c28ec807dd24293afa3dceede0da3b9 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/1c28ec807dd24293afa3dceede0da3b9 2024-11-15T14:17:00,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c6de1cbed69d41ae950f1af23726dac5 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/c6de1cbed69d41ae950f1af23726dac5 2024-11-15T14:17:00,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/a0c724b20d284102a6b9b3739c621e12 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/a0c724b20d284102a6b9b3739c621e12 2024-11-15T14:17:00,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/863c098aea4145c5b05ea84aa5374be2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/2ff6b3d0afc94f58ab46bcb0738f3150, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b6607b492ae449b38858668d6eb199cf, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b5f5cee106074fc58098a8587cf1b0b7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0d0cce1026d94c0982e20b5945ba65e4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/209091333d034fb5b1321c545221a2c7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0eddae22ec9c43069915069f1a50a6d6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/7fc382c553c243cc874df8b92612a361, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/340327dd8df44ac084c75369fefbbca7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/101ed7b2c1844de993052f80ba9775f6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0cc48cb5cee54b49b254895ec08f377e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/8817a1c2dc294b198f637d8fd0d020e4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/2756d52112a34482b85e58fdf1610e54, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a18a8c9bc27e445195978da308e7ab57, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e3129a0f6d8c4b419b2803d1985f9d1e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/96c840fa71424d1ea338966fb90750da, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/127a0717b4e94d7f8777e35b48afea33, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b837c0c2b6f5452e98c63d148bd0fede, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/6ad58383e37d4b3eb65349da24755527, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e9deb780fb804fbba06a02519988da35, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/487dfe36a57d4acb8f72cc3db8bb32be, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a943d3160e044c7990055fbc8b8fa7c2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/9cbbbd3aed4143db937f8cf4566ca6dd, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a5a79865e1cc4c239d6e4fa26b770769, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/308342621e384b88b7159f5893e5106c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/55fe49fa604049948c4f81e45e1d19ab, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b92ddbaa2a0840c491a0ffb5fe46fe73, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/60eb4d36971949ab9eff283aa4c8d666] to archive 2024-11-15T14:17:00,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:17:00,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/863c098aea4145c5b05ea84aa5374be2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/863c098aea4145c5b05ea84aa5374be2 2024-11-15T14:17:00,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/2ff6b3d0afc94f58ab46bcb0738f3150 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/2ff6b3d0afc94f58ab46bcb0738f3150 2024-11-15T14:17:00,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b6607b492ae449b38858668d6eb199cf to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b6607b492ae449b38858668d6eb199cf 2024-11-15T14:17:00,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b5f5cee106074fc58098a8587cf1b0b7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b5f5cee106074fc58098a8587cf1b0b7 2024-11-15T14:17:00,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0d0cce1026d94c0982e20b5945ba65e4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0d0cce1026d94c0982e20b5945ba65e4 2024-11-15T14:17:00,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/209091333d034fb5b1321c545221a2c7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/209091333d034fb5b1321c545221a2c7 2024-11-15T14:17:00,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0eddae22ec9c43069915069f1a50a6d6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0eddae22ec9c43069915069f1a50a6d6 2024-11-15T14:17:00,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/7fc382c553c243cc874df8b92612a361 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/7fc382c553c243cc874df8b92612a361 2024-11-15T14:17:00,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/340327dd8df44ac084c75369fefbbca7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/340327dd8df44ac084c75369fefbbca7 2024-11-15T14:17:00,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/101ed7b2c1844de993052f80ba9775f6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/101ed7b2c1844de993052f80ba9775f6 2024-11-15T14:17:00,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0cc48cb5cee54b49b254895ec08f377e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/0cc48cb5cee54b49b254895ec08f377e 2024-11-15T14:17:00,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/8817a1c2dc294b198f637d8fd0d020e4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/8817a1c2dc294b198f637d8fd0d020e4 2024-11-15T14:17:00,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/2756d52112a34482b85e58fdf1610e54 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/2756d52112a34482b85e58fdf1610e54 2024-11-15T14:17:00,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a18a8c9bc27e445195978da308e7ab57 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a18a8c9bc27e445195978da308e7ab57 2024-11-15T14:17:00,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e3129a0f6d8c4b419b2803d1985f9d1e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e3129a0f6d8c4b419b2803d1985f9d1e 2024-11-15T14:17:00,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/96c840fa71424d1ea338966fb90750da to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/96c840fa71424d1ea338966fb90750da 2024-11-15T14:17:00,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/127a0717b4e94d7f8777e35b48afea33 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/127a0717b4e94d7f8777e35b48afea33 2024-11-15T14:17:00,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b837c0c2b6f5452e98c63d148bd0fede to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b837c0c2b6f5452e98c63d148bd0fede 2024-11-15T14:17:00,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/6ad58383e37d4b3eb65349da24755527 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/6ad58383e37d4b3eb65349da24755527 2024-11-15T14:17:00,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e9deb780fb804fbba06a02519988da35 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/e9deb780fb804fbba06a02519988da35 2024-11-15T14:17:00,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/487dfe36a57d4acb8f72cc3db8bb32be to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/487dfe36a57d4acb8f72cc3db8bb32be 2024-11-15T14:17:00,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a943d3160e044c7990055fbc8b8fa7c2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a943d3160e044c7990055fbc8b8fa7c2 2024-11-15T14:17:00,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/9cbbbd3aed4143db937f8cf4566ca6dd to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/9cbbbd3aed4143db937f8cf4566ca6dd 2024-11-15T14:17:00,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a5a79865e1cc4c239d6e4fa26b770769 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/a5a79865e1cc4c239d6e4fa26b770769 2024-11-15T14:17:00,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/308342621e384b88b7159f5893e5106c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/308342621e384b88b7159f5893e5106c 2024-11-15T14:17:00,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/55fe49fa604049948c4f81e45e1d19ab to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/55fe49fa604049948c4f81e45e1d19ab 2024-11-15T14:17:00,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b92ddbaa2a0840c491a0ffb5fe46fe73 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/b92ddbaa2a0840c491a0ffb5fe46fe73 2024-11-15T14:17:00,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/60eb4d36971949ab9eff283aa4c8d666 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/60eb4d36971949ab9eff283aa4c8d666 2024-11-15T14:17:00,716 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/recovered.edits/500.seqid, newMaxSeqId=500, maxSeqId=1 2024-11-15T14:17:00,717 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. 2024-11-15T14:17:00,717 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for f24e805c31c506536abbd73db35b0c0f: Waiting for close lock at 1731680219785Running coprocessor pre-close hooks at 1731680219785Disabling compacts and flushes for region at 1731680219785Disabling writes for close at 1731680219785Obtaining lock to block concurrent updates at 1731680219785Preparing flush snapshotting stores in f24e805c31c506536abbd73db35b0c0f at 1731680219785Finished memstore snapshotting TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731680219786 (+1 ms)Flushing stores of TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f. at 1731680219787 (+1 ms)Flushing f24e805c31c506536abbd73db35b0c0f/A: creating writer at 1731680219787Flushing f24e805c31c506536abbd73db35b0c0f/A: appending metadata at 1731680219791 (+4 ms)Flushing f24e805c31c506536abbd73db35b0c0f/A: closing flushed file at 1731680219791Flushing f24e805c31c506536abbd73db35b0c0f/B: creating writer at 1731680220205 (+414 ms)Flushing f24e805c31c506536abbd73db35b0c0f/B: appending metadata at 1731680220208 (+3 ms)Flushing f24e805c31c506536abbd73db35b0c0f/B: closing flushed file at 1731680220208Flushing f24e805c31c506536abbd73db35b0c0f/C: creating writer at 1731680220621 (+413 ms)Flushing f24e805c31c506536abbd73db35b0c0f/C: appending metadata at 1731680220626 (+5 ms)Flushing f24e805c31c506536abbd73db35b0c0f/C: closing flushed file at 1731680220626Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5634c0f8: reopening flushed file at 1731680220634 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d24bc8c: reopening flushed file at 1731680220638 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@761aea43: reopening flushed file at 1731680220642 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f24e805c31c506536abbd73db35b0c0f in 862ms, sequenceid=497, compaction requested=true at 1731680220647 (+5 ms)Writing region close event to WAL at 1731680220714 (+67 ms)Running coprocessor post-close hooks at 1731680220717 (+3 ms)Closed at 1731680220717 2024-11-15T14:17:00,719 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:17:00,720 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=f24e805c31c506536abbd73db35b0c0f, regionState=CLOSED 2024-11-15T14:17:00,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:17:00,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-15T14:17:00,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure f24e805c31c506536abbd73db35b0c0f, server=26754e923c4a,40177,1731680077530 in 1.0910 sec 2024-11-15T14:17:00,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-15T14:17:00,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f24e805c31c506536abbd73db35b0c0f, UNASSIGN in 1.0970 sec 2024-11-15T14:17:00,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-15T14:17:00,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1020 sec 2024-11-15T14:17:00,731 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680220730"}]},"ts":"1731680220730"} 2024-11-15T14:17:00,732 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T14:17:00,732 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T14:17:00,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1160 sec 2024-11-15T14:17:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T14:17:00,761 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:00,762 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T14:17:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:00,764 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T14:17:00,764 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:00,767 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:17:00,769 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/recovered.edits] 2024-11-15T14:17:00,771 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/7468cef0033a4d10a87cd4f83f0eeee3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/7468cef0033a4d10a87cd4f83f0eeee3 2024-11-15T14:17:00,772 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d3592fd64c7e48f1a188f045eb47c6ab to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d3592fd64c7e48f1a188f045eb47c6ab 2024-11-15T14:17:00,772 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d8044720e8124d5d820539f5f8b7af23 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/d8044720e8124d5d820539f5f8b7af23 2024-11-15T14:17:00,773 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/fbbcd8e75bfb4c2192e6bccd09c792fc to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/A/fbbcd8e75bfb4c2192e6bccd09c792fc 2024-11-15T14:17:00,775 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/01b0e89ac5434ad7b0b71c8ce88e3afc to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/01b0e89ac5434ad7b0b71c8ce88e3afc 2024-11-15T14:17:00,775 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/21d7a246f35d4e1f80e11aa82e41a4fa to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/21d7a246f35d4e1f80e11aa82e41a4fa 2024-11-15T14:17:00,776 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7d570564694445d59128b9eb1e204a99 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/7d570564694445d59128b9eb1e204a99 2024-11-15T14:17:00,777 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/e70fb6adf09a4700b6c3ab78329607f2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/B/e70fb6adf09a4700b6c3ab78329607f2 2024-11-15T14:17:00,778 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/1aed7f6f87f14b5092bfa5ec6b79fd41 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/1aed7f6f87f14b5092bfa5ec6b79fd41 2024-11-15T14:17:00,779 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/31926f4c6a9c4026a55d4151b5f2de86 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/31926f4c6a9c4026a55d4151b5f2de86 2024-11-15T14:17:00,780 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/417ddcad532648349091b9d7e9f965cb to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/417ddcad532648349091b9d7e9f965cb 2024-11-15T14:17:00,780 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/7e1b445e192048ef8695f6b50019b733 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/C/7e1b445e192048ef8695f6b50019b733 2024-11-15T14:17:00,782 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/recovered.edits/500.seqid to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f/recovered.edits/500.seqid 2024-11-15T14:17:00,782 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/f24e805c31c506536abbd73db35b0c0f 2024-11-15T14:17:00,782 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T14:17:00,784 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:00,786 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T14:17:00,788 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T14:17:00,789 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:00,789 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T14:17:00,789 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731680220789"}]},"ts":"9223372036854775807"} 2024-11-15T14:17:00,791 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T14:17:00,791 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f24e805c31c506536abbd73db35b0c0f, NAME => 'TestAcidGuarantees,,1731680192222.f24e805c31c506536abbd73db35b0c0f.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T14:17:00,791 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T14:17:00,791 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731680220791"}]},"ts":"9223372036854775807"} 2024-11-15T14:17:00,793 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T14:17:00,794 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:00,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-15T14:17:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T14:17:00,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T14:17:00,872 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:00,885 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 252), OpenFileDescriptor=447 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=814 (was 802) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=534 (was 713) 2024-11-15T14:17:00,893 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=814, ProcessCount=11, AvailableMemoryMB=533 2024-11-15T14:17:00,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T14:17:00,894 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T14:17:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:00,896 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T14:17:00,896 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:00,896 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-15T14:17:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T14:17:00,898 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T14:17:00,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742376_1552 (size=1061) 2024-11-15T14:17:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T14:17:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T14:17:01,307 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931 2024-11-15T14:17:01,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742377_1553 (size=53) 2024-11-15T14:17:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T14:17:01,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:17:01,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e34e905c516b4c17b3db18acbd12c2a8, disabling compactions & flushes 2024-11-15T14:17:01,715 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:01,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:01,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. after waiting 0 ms 2024-11-15T14:17:01,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:01,715 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:01,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e34e905c516b4c17b3db18acbd12c2a8: Waiting for close lock at 1731680221715Disabling compacts and flushes for region at 1731680221715Disabling writes for close at 1731680221715Writing region close event to WAL at 1731680221715Closed at 1731680221715 2024-11-15T14:17:01,716 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T14:17:01,717 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731680221716"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731680221716"}]},"ts":"1731680221716"} 2024-11-15T14:17:01,718 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T14:17:01,718 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T14:17:01,719 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680221719"}]},"ts":"1731680221719"} 2024-11-15T14:17:01,720 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T14:17:01,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, ASSIGN}] 2024-11-15T14:17:01,721 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, ASSIGN 2024-11-15T14:17:01,721 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, ASSIGN; state=OFFLINE, location=26754e923c4a,40177,1731680077530; forceNewPlan=false, retain=false 2024-11-15T14:17:01,873 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=e34e905c516b4c17b3db18acbd12c2a8, regionState=OPENING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:17:01,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, ASSIGN because future has completed 2024-11-15T14:17:01,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:17:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T14:17:02,036 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:02,036 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} 2024-11-15T14:17:02,037 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:02,037 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:17:02,037 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:02,037 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:02,039 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:02,041 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:17:02,042 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e34e905c516b4c17b3db18acbd12c2a8 columnFamilyName A 2024-11-15T14:17:02,042 DEBUG [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:02,043 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.HStore(327): Store=e34e905c516b4c17b3db18acbd12c2a8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:17:02,043 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:02,044 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:17:02,045 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e34e905c516b4c17b3db18acbd12c2a8 columnFamilyName B 2024-11-15T14:17:02,045 DEBUG [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:02,046 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.HStore(327): Store=e34e905c516b4c17b3db18acbd12c2a8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:17:02,046 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:02,047 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:17:02,047 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e34e905c516b4c17b3db18acbd12c2a8 columnFamilyName C 2024-11-15T14:17:02,047 DEBUG [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:02,048 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.HStore(327): Store=e34e905c516b4c17b3db18acbd12c2a8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:17:02,048 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:02,049 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:02,050 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:02,050 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:02,052 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:02,052 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:02,053 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T14:17:02,055 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:02,058 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T14:17:02,059 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened e34e905c516b4c17b3db18acbd12c2a8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61185045, jitterRate=-0.08827178180217743}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T14:17:02,059 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:02,060 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for e34e905c516b4c17b3db18acbd12c2a8: Running coprocessor pre-open hook at 1731680222037Writing region info on filesystem at 1731680222037Initializing all the Stores at 1731680222039 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680222039Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680222039Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680222039Cleaning up temporary data from old regions at 1731680222052 (+13 ms)Running coprocessor post-open hooks at 1731680222059 (+7 ms)Region opened successfully at 1731680222060 (+1 ms) 2024-11-15T14:17:02,061 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., pid=144, masterSystemTime=1731680222032 2024-11-15T14:17:02,062 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:02,062 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:02,063 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=e34e905c516b4c17b3db18acbd12c2a8, regionState=OPEN, openSeqNum=2, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:17:02,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:17:02,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-15T14:17:02,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 in 188 msec 2024-11-15T14:17:02,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-15T14:17:02,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, ASSIGN in 347 msec 2024-11-15T14:17:02,069 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T14:17:02,070 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680222069"}]},"ts":"1731680222069"} 2024-11-15T14:17:02,071 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T14:17:02,072 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T14:17:02,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1770 sec 2024-11-15T14:17:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-15T14:17:03,043 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:03,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:03,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:17:03,044 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:17:03,045 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:17:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:17:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:17:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1267c36c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:17:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:17:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:03,048 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:17:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@249d0c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:03,049 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:17:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:17:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:17:03,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:17:03,053 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T14:17:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 26754e923c4a,35807,1731680075796 2024-11-15T14:17:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@264bef38 2024-11-15T14:17:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T14:17:03,055 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T14:17:03,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T14:17:03,056 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T14:17:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:03,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742378_1554 (size=1097) 2024-11-15T14:17:03,469 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T14:17:03,470 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T14:17:03,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T14:17:03,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, REOPEN/MOVE}] 2024-11-15T14:17:03,478 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, REOPEN/MOVE 2024-11-15T14:17:03,478 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e34e905c516b4c17b3db18acbd12c2a8, regionState=CLOSING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:17:03,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, REOPEN/MOVE because future has completed 2024-11-15T14:17:03,480 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T14:17:03,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:17:03,633 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,633 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T14:17:03,633 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing e34e905c516b4c17b3db18acbd12c2a8, disabling compactions & flushes 2024-11-15T14:17:03,633 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:03,633 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:03,633 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. after waiting 0 ms 2024-11-15T14:17:03,633 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:03,640 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T14:17:03,641 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:03,641 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for e34e905c516b4c17b3db18acbd12c2a8: Waiting for close lock at 1731680223633Running coprocessor pre-close hooks at 1731680223633Disabling compacts and flushes for region at 1731680223633Disabling writes for close at 1731680223633Writing region close event to WAL at 1731680223634 (+1 ms)Running coprocessor post-close hooks at 1731680223641 (+7 ms)Closed at 1731680223641 2024-11-15T14:17:03,642 WARN [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: e34e905c516b4c17b3db18acbd12c2a8 to self. 2024-11-15T14:17:03,645 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,646 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e34e905c516b4c17b3db18acbd12c2a8, regionState=CLOSED 2024-11-15T14:17:03,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:17:03,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-15T14:17:03,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 in 169 msec 2024-11-15T14:17:03,653 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, REOPEN/MOVE; state=CLOSED, location=26754e923c4a,40177,1731680077530; forceNewPlan=false, retain=true 2024-11-15T14:17:03,804 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e34e905c516b4c17b3db18acbd12c2a8, regionState=OPENING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:17:03,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, REOPEN/MOVE because future has completed 2024-11-15T14:17:03,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:17:03,964 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:03,964 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} 2024-11-15T14:17:03,965 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,965 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T14:17:03,965 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,965 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,968 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,969 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:17:03,969 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e34e905c516b4c17b3db18acbd12c2a8 columnFamilyName A 2024-11-15T14:17:03,971 DEBUG [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:03,971 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.HStore(327): Store=e34e905c516b4c17b3db18acbd12c2a8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:17:03,972 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,972 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:17:03,973 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e34e905c516b4c17b3db18acbd12c2a8 columnFamilyName B 2024-11-15T14:17:03,973 DEBUG [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:03,973 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.HStore(327): Store=e34e905c516b4c17b3db18acbd12c2a8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:17:03,973 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,974 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T14:17:03,974 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e34e905c516b4c17b3db18acbd12c2a8 columnFamilyName C 2024-11-15T14:17:03,974 DEBUG [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:03,975 INFO [StoreOpener-e34e905c516b4c17b3db18acbd12c2a8-1 {}] regionserver.HStore(327): Store=e34e905c516b4c17b3db18acbd12c2a8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T14:17:03,975 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:03,975 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,975 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,976 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,977 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,977 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,977 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T14:17:03,978 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,979 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened e34e905c516b4c17b3db18acbd12c2a8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68847405, jitterRate=0.025906279683113098}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T14:17:03,979 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:03,979 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for e34e905c516b4c17b3db18acbd12c2a8: Running coprocessor pre-open hook at 1731680223965Writing region info on filesystem at 1731680223965Initializing all the Stores at 1731680223967 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680223967Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680223967Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731680223967Cleaning up temporary data from old regions at 1731680223977 (+10 ms)Running coprocessor post-open hooks at 1731680223979 (+2 ms)Region opened successfully at 1731680223979 2024-11-15T14:17:03,980 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., pid=149, masterSystemTime=1731680223957 2024-11-15T14:17:03,981 DEBUG [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:03,981 INFO [RS_OPEN_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:03,982 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e34e905c516b4c17b3db18acbd12c2a8, regionState=OPEN, openSeqNum=5, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:17:03,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:17:03,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-15T14:17:03,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 in 177 msec 2024-11-15T14:17:03,986 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-15T14:17:03,986 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, REOPEN/MOVE in 507 msec 2024-11-15T14:17:03,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-15T14:17:03,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-11-15T14:17:03,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 932 msec 2024-11-15T14:17:03,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-15T14:17:03,991 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:03,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:03,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:17:03,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:17:03,992 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:17:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:17:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:17:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@578986da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:17:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:17:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:03,993 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:17:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5065b43f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:03,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:03,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:17:03,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:17:03,995 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:17:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:17:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:17:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:17:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:17:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:03,996 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:17:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25265084, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:03,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:03,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:17:03,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:17:03,998 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:17:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:17:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:17:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c0cbdb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:17:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:17:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:03,999 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:17:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5adebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,000 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:17:04,000 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:17:04,000 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:17:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:17:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:17:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d8e1d16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:17:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:17:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:04,001 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:17:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e326eac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:17:04,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:17:04,003 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:17:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:17:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:17:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4edbb444, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:17:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:17:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:04,004 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:17:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b996902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:17:04,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:17:04,005 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:17:04,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:17:04,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:17:04,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24b03917, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:17:04,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:17:04,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:04,006 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:17:04,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b16cd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,007 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:17:04,007 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:17:04,008 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:17:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:17:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:17:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f11cd96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:17:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:17:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:04,009 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:17:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2afbf675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:17:04,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:17:04,010 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:17:04,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:17:04,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:17:04,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c19fa12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:17:04,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:17:04,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:04,011 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:17:04,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2859e728, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:17:04,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:17:04,013 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:17:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:17:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:17:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa52fd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:17:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:17:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:04,014 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:17:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46dcebc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26754e923c4a,35807,-1 for getting cluster id 2024-11-15T14:17:04,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T14:17:04,016 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f9dcf31d-c076-4b78-aeaf-342511b996a6' 2024-11-15T14:17:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T14:17:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f9dcf31d-c076-4b78-aeaf-342511b996a6" 2024-11-15T14:17:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fa684eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26754e923c4a,35807,-1] 2024-11-15T14:17:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T14:17:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:04,017 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T14:17:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b1f1cdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T14:17:04,018 DEBUG [Thread-2473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:17:04,018 DEBUG [Thread-2471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:17:04,018 DEBUG [Thread-2464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:17:04,019 DEBUG [Thread-2462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:17:04,019 DEBUG [Thread-2458 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:17:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:17:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:17:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:17:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:17:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:17:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:17:04,019 DEBUG [Thread-2477 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:17:04,019 DEBUG [Thread-2466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:17:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:17:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:17:04,020 DEBUG [Thread-2469 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:17:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:17:04,020 DEBUG [Thread-2460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:17:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:17:04,020 DEBUG [Thread-2477 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:17:04,020 DEBUG [Thread-2475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T14:17:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:17:04,020 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:17:04,021 DEBUG [Thread-2469 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:17:04,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:17:04,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:17:04,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:17:04,021 DEBUG [Thread-2469 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:17:04,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:17:04,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:17:04,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:17:04,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:17:04,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:17:04,022 DEBUG [Thread-2477 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:17:04,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:17:04,022 DEBUG [Thread-2466 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26754e923c4a,40177,1731680077530, seqNum=-1] 2024-11-15T14:17:04,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:17:04,022 DEBUG [Thread-2466 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T14:17:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:17:04,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:17:04,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:17:04,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:17:04,024 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:17:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:17:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:17:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:17:04,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T14:17:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:17:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:17:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-15T14:17:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:17:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5] 2024-11-15T14:17:04,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T14:17:04,027 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:17:04,028 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:17:04,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:17:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:04,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T14:17:04,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:04,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:04,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:04,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:04,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:04,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:04,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680284047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731680224045, max timestamp=1731680224049Num uniques 859; 2024-11-15T14:17:04,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:17:04,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731680224045, max timestamp=1731680224049Num uniques 859; 2024-11-15T14:17:04,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:17:04,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731680224045, max timestamp=1731680224049Num uniques 859; 2024-11-15T14:17:04,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8102, suffix heap size=117064,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=177 2024-11-15T14:17:04,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152a94246fc5b24a0fad66ee58ef93dcc3_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680224028/Put/seqid=0 2024-11-15T14:17:04,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742379_1555 (size=12154) 2024-11-15T14:17:04,059 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:04,061 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152a94246fc5b24a0fad66ee58ef93dcc3_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152a94246fc5b24a0fad66ee58ef93dcc3_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:04,062 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/d5a81745ab254d9d91eb3c33ab08f9d3, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:04,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/d5a81745ab254d9d91eb3c33ab08f9d3 is 175, key is test_row_0/A:col10/1731680224028/Put/seqid=0 2024-11-15T14:17:04,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742380_1556 (size=30955) 2024-11-15T14:17:04,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680284073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680284072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680284074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680284075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T14:17:04,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680284162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-15T14:17:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:04,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680284181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680284181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680284181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680284192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-15T14:17:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:04,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T14:17:04,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680284371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680284392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680284392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680284392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680284402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,466 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/d5a81745ab254d9d91eb3c33ab08f9d3 2024-11-15T14:17:04,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-15T14:17:04,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:04,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:04,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:04,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/841949ef840f40c69164e10e389d49ec is 50, key is test_row_0/B:col10/1731680224028/Put/seqid=0 2024-11-15T14:17:04,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742381_1557 (size=12001) 2024-11-15T14:17:04,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/841949ef840f40c69164e10e389d49ec 2024-11-15T14:17:04,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/e35d32edb51d4f4c9460ac52ed2d88cb is 50, key is test_row_0/C:col10/1731680224028/Put/seqid=0 2024-11-15T14:17:04,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742382_1558 (size=12001) 2024-11-15T14:17:04,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-15T14:17:04,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:04,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:04,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:04,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T14:17:04,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680284681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680284701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680284702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680284702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680284712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:04,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-15T14:17:04,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:04,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:04,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:04,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,812 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T14:17:04,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-15T14:17:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:04,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:04,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/e35d32edb51d4f4c9460ac52ed2d88cb 2024-11-15T14:17:04,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/d5a81745ab254d9d91eb3c33ab08f9d3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/d5a81745ab254d9d91eb3c33ab08f9d3 2024-11-15T14:17:05,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/d5a81745ab254d9d91eb3c33ab08f9d3, entries=150, sequenceid=15, filesize=30.2 K 2024-11-15T14:17:05,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/841949ef840f40c69164e10e389d49ec as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/841949ef840f40c69164e10e389d49ec 2024-11-15T14:17:05,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/841949ef840f40c69164e10e389d49ec, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T14:17:05,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/e35d32edb51d4f4c9460ac52ed2d88cb as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e35d32edb51d4f4c9460ac52ed2d88cb 2024-11-15T14:17:05,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e35d32edb51d4f4c9460ac52ed2d88cb, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T14:17:05,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.04 KB/129068 for e34e905c516b4c17b3db18acbd12c2a8 in 1013ms, sequenceid=15, compaction requested=false 2024-11-15T14:17:05,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:05,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-15T14:17:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:05,108 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=126.04 KB heapSize=357.13 KB 2024-11-15T14:17:05,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:05,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:17:05,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:05,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:17:05,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:05,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:17:05,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bf37d0e22370404fb1fa48e36b8b2c80_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680224072/Put/seqid=0 2024-11-15T14:17:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T14:17:05,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742383_1559 (size=12154) 2024-11-15T14:17:05,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:05,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:05,207 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bf37d0e22370404fb1fa48e36b8b2c80_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf37d0e22370404fb1fa48e36b8b2c80_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:05,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/1733618375f54c6fbdc6542fd4e8e242, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:05,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/1733618375f54c6fbdc6542fd4e8e242 is 175, key is test_row_0/A:col10/1731680224072/Put/seqid=0 2024-11-15T14:17:05,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680285223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680285230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680285229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680285235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680285235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742384_1560 (size=30955) 2024-11-15T14:17:05,256 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/1733618375f54c6fbdc6542fd4e8e242 2024-11-15T14:17:05,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/4f3cd6fbf1704423a1d040f524a7fc30 is 50, key is test_row_0/B:col10/1731680224072/Put/seqid=0 2024-11-15T14:17:05,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680285343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742385_1561 (size=12001) 2024-11-15T14:17:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,347 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/4f3cd6fbf1704423a1d040f524a7fc30 2024-11-15T14:17:05,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680285352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680285352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680285352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680285355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/4b5fa06742034bfeb0556d2f1a45216b is 50, key is test_row_0/C:col10/1731680224072/Put/seqid=0 2024-11-15T14:17:05,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742386_1562 (size=12001) 2024-11-15T14:17:05,484 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.06 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/4b5fa06742034bfeb0556d2f1a45216b 2024-11-15T14:17:05,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/1733618375f54c6fbdc6542fd4e8e242 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/1733618375f54c6fbdc6542fd4e8e242 2024-11-15T14:17:05,548 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/1733618375f54c6fbdc6542fd4e8e242, entries=150, sequenceid=50, filesize=30.2 K 2024-11-15T14:17:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/4f3cd6fbf1704423a1d040f524a7fc30 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/4f3cd6fbf1704423a1d040f524a7fc30 2024-11-15T14:17:05,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680285552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680285562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680285562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680285563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680285571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,600 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/4f3cd6fbf1704423a1d040f524a7fc30, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T14:17:05,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/4b5fa06742034bfeb0556d2f1a45216b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/4b5fa06742034bfeb0556d2f1a45216b 2024-11-15T14:17:05,640 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/4b5fa06742034bfeb0556d2f1a45216b, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T14:17:05,645 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~126.04 KB/129068, heapSize ~357.08 KB/365648, currentSize=60.38 KB/61830 for e34e905c516b4c17b3db18acbd12c2a8 in 537ms, sequenceid=50, compaction requested=false 2024-11-15T14:17:05,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:05,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:05,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-15T14:17:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-15T14:17:05,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-15T14:17:05,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6260 sec 2024-11-15T14:17:05,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.6340 sec 2024-11-15T14:17:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:05,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T14:17:05,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:05,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:05,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:05,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:05,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:05,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:05,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111526925c4196064505b1aa347dd31cdafe_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680225865/Put/seqid=0 2024-11-15T14:17:05,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680285913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680285917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680285919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680285919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680285921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:05,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:05,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742387_1563 (size=12154) 2024-11-15T14:17:06,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680286032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680286032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680286032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680286032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680286042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T14:17:06,172 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:06,173 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:17:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-15T14:17:06,175 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:17:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T14:17:06,176 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:17:06,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:17:06,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680286242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680286252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680286252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680286252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,265 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T14:17:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T14:17:06,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680286282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T14:17:06,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:06,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:06,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:06,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:06,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:06,387 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:06,407 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111526925c4196064505b1aa347dd31cdafe_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111526925c4196064505b1aa347dd31cdafe_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:06,415 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/6484132b33da456db498ee9750c426e0, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:06,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/6484132b33da456db498ee9750c426e0 is 175, key is test_row_0/A:col10/1731680225865/Put/seqid=0 2024-11-15T14:17:06,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742388_1564 (size=30955) 2024-11-15T14:17:06,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T14:17:06,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:06,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:06,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:06,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:06,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:06,491 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/6484132b33da456db498ee9750c426e0 2024-11-15T14:17:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T14:17:06,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/163e6f766e8d4dcf87007c6341256d02 is 50, key is test_row_0/B:col10/1731680225865/Put/seqid=0 2024-11-15T14:17:06,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742389_1565 (size=12001) 2024-11-15T14:17:06,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/163e6f766e8d4dcf87007c6341256d02 2024-11-15T14:17:06,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680286553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680286562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680286562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680286562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/a2cc4f8cba6847bd9f3077bfd996f706 is 50, key is test_row_0/C:col10/1731680225865/Put/seqid=0 2024-11-15T14:17:06,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680286592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:06,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742390_1566 (size=12001) 2024-11-15T14:17:06,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/a2cc4f8cba6847bd9f3077bfd996f706 2024-11-15T14:17:06,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T14:17:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:06,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:06,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/6484132b33da456db498ee9750c426e0 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/6484132b33da456db498ee9750c426e0 2024-11-15T14:17:06,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/6484132b33da456db498ee9750c426e0, entries=150, sequenceid=63, filesize=30.2 K 2024-11-15T14:17:06,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/163e6f766e8d4dcf87007c6341256d02 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/163e6f766e8d4dcf87007c6341256d02 2024-11-15T14:17:06,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/163e6f766e8d4dcf87007c6341256d02, entries=150, sequenceid=63, filesize=11.7 K 2024-11-15T14:17:06,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/a2cc4f8cba6847bd9f3077bfd996f706 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/a2cc4f8cba6847bd9f3077bfd996f706 2024-11-15T14:17:06,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/a2cc4f8cba6847bd9f3077bfd996f706, entries=150, sequenceid=63, filesize=11.7 K 2024-11-15T14:17:06,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e34e905c516b4c17b3db18acbd12c2a8 in 821ms, sequenceid=63, compaction requested=true 2024-11-15T14:17:06,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:06,689 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:06,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:17:06,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:06,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:17:06,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:06,690 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:06,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:17:06,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:06,696 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:06,696 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/A is initiating minor compaction (all files) 2024-11-15T14:17:06,696 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/A in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:06,696 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/d5a81745ab254d9d91eb3c33ab08f9d3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/1733618375f54c6fbdc6542fd4e8e242, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/6484132b33da456db498ee9750c426e0] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=90.7 K 2024-11-15T14:17:06,696 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:06,697 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/d5a81745ab254d9d91eb3c33ab08f9d3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/1733618375f54c6fbdc6542fd4e8e242, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/6484132b33da456db498ee9750c426e0] 2024-11-15T14:17:06,697 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:06,697 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/B is initiating minor compaction (all files) 2024-11-15T14:17:06,697 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/B in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:06,697 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5a81745ab254d9d91eb3c33ab08f9d3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731680224024 2024-11-15T14:17:06,697 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/841949ef840f40c69164e10e389d49ec, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/4f3cd6fbf1704423a1d040f524a7fc30, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/163e6f766e8d4dcf87007c6341256d02] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=35.2 K 2024-11-15T14:17:06,697 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 841949ef840f40c69164e10e389d49ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731680224024 2024-11-15T14:17:06,698 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1733618375f54c6fbdc6542fd4e8e242, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731680224070 2024-11-15T14:17:06,698 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f3cd6fbf1704423a1d040f524a7fc30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731680224070 2024-11-15T14:17:06,698 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6484132b33da456db498ee9750c426e0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731680225229 2024-11-15T14:17:06,698 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 163e6f766e8d4dcf87007c6341256d02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731680225229 2024-11-15T14:17:06,714 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#B#compaction#487 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:06,715 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/03e15459b2764977903f1066d6df6dde is 50, key is test_row_0/B:col10/1731680225865/Put/seqid=0 2024-11-15T14:17:06,720 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:06,739 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115468d30a15da44af6861dbd76704dba7c_e34e905c516b4c17b3db18acbd12c2a8 store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:06,742 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115468d30a15da44af6861dbd76704dba7c_e34e905c516b4c17b3db18acbd12c2a8, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:06,742 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115468d30a15da44af6861dbd76704dba7c_e34e905c516b4c17b3db18acbd12c2a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:06,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742391_1567 (size=12104) 2024-11-15T14:17:06,778 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/03e15459b2764977903f1066d6df6dde as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/03e15459b2764977903f1066d6df6dde 2024-11-15T14:17:06,788 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/B of e34e905c516b4c17b3db18acbd12c2a8 into 03e15459b2764977903f1066d6df6dde(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:06,788 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:06,788 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/B, priority=13, startTime=1731680226690; duration=0sec 2024-11-15T14:17:06,788 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:06,788 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:B 2024-11-15T14:17:06,789 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:06,790 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:06,790 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/C is initiating minor compaction (all files) 2024-11-15T14:17:06,790 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/C in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:06,790 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e35d32edb51d4f4c9460ac52ed2d88cb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/4b5fa06742034bfeb0556d2f1a45216b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/a2cc4f8cba6847bd9f3077bfd996f706] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=35.2 K 2024-11-15T14:17:06,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742392_1568 (size=4469) 2024-11-15T14:17:06,791 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting e35d32edb51d4f4c9460ac52ed2d88cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731680224024 2024-11-15T14:17:06,791 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b5fa06742034bfeb0556d2f1a45216b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731680224070 2024-11-15T14:17:06,791 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting a2cc4f8cba6847bd9f3077bfd996f706, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731680225229 2024-11-15T14:17:06,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T14:17:06,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:06,796 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T14:17:06,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:06,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:06,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:06,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:06,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:06,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:06,800 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#C#compaction#489 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:06,800 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/4523ff94c3654eb294fb55cb4cb5ce8b is 50, key is test_row_0/C:col10/1731680225865/Put/seqid=0 2024-11-15T14:17:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T14:17:06,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742393_1569 (size=12104) 2024-11-15T14:17:06,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159b618aeddbc8475080a7fca693f02451_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680225918/Put/seqid=0 2024-11-15T14:17:06,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742394_1570 (size=12154) 2024-11-15T14:17:06,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:06,860 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159b618aeddbc8475080a7fca693f02451_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159b618aeddbc8475080a7fca693f02451_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:06,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/94abea38d5a74b3ab9d0a98dd8d7b07d, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:06,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/94abea38d5a74b3ab9d0a98dd8d7b07d is 175, key is test_row_0/A:col10/1731680225918/Put/seqid=0 2024-11-15T14:17:06,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742395_1571 (size=30955) 2024-11-15T14:17:06,866 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/94abea38d5a74b3ab9d0a98dd8d7b07d 2024-11-15T14:17:06,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/fb1a32034b3e491bb536d654a33c9494 is 50, key is test_row_0/B:col10/1731680225918/Put/seqid=0 2024-11-15T14:17:06,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742396_1572 (size=12001) 2024-11-15T14:17:07,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:07,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680287090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680287093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680287092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680287097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680287121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,192 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#A#compaction#488 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:07,192 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/913d1123963d44af9bef5ea03a8e5178 is 175, key is test_row_0/A:col10/1731680225865/Put/seqid=0 2024-11-15T14:17:07,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742397_1573 (size=31058) 2024-11-15T14:17:07,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680287202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680287203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680287203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680287203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,214 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/4523ff94c3654eb294fb55cb4cb5ce8b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/4523ff94c3654eb294fb55cb4cb5ce8b 2024-11-15T14:17:07,225 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/C of e34e905c516b4c17b3db18acbd12c2a8 into 4523ff94c3654eb294fb55cb4cb5ce8b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:07,225 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:07,225 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/C, priority=13, startTime=1731680226690; duration=0sec 2024-11-15T14:17:07,225 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:07,225 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:C 2024-11-15T14:17:07,297 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/fb1a32034b3e491bb536d654a33c9494 2024-11-15T14:17:07,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/37ac3cf93bf14c8886f5c87066cd7038 is 50, key is test_row_0/C:col10/1731680225918/Put/seqid=0 2024-11-15T14:17:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T14:17:07,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742398_1574 (size=12001) 2024-11-15T14:17:07,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680287412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680287412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680287412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680287412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,610 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/913d1123963d44af9bef5ea03a8e5178 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/913d1123963d44af9bef5ea03a8e5178 2024-11-15T14:17:07,620 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/A of e34e905c516b4c17b3db18acbd12c2a8 into 913d1123963d44af9bef5ea03a8e5178(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:07,620 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:07,620 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/A, priority=13, startTime=1731680226689; duration=0sec 2024-11-15T14:17:07,621 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:07,621 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:A 2024-11-15T14:17:07,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680287722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680287722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680287722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680287722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:07,744 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/37ac3cf93bf14c8886f5c87066cd7038 2024-11-15T14:17:07,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/94abea38d5a74b3ab9d0a98dd8d7b07d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/94abea38d5a74b3ab9d0a98dd8d7b07d 2024-11-15T14:17:07,756 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/94abea38d5a74b3ab9d0a98dd8d7b07d, entries=150, sequenceid=83, filesize=30.2 K 2024-11-15T14:17:07,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/fb1a32034b3e491bb536d654a33c9494 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/fb1a32034b3e491bb536d654a33c9494 2024-11-15T14:17:07,761 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/fb1a32034b3e491bb536d654a33c9494, entries=150, sequenceid=83, filesize=11.7 K 2024-11-15T14:17:07,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/37ac3cf93bf14c8886f5c87066cd7038 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/37ac3cf93bf14c8886f5c87066cd7038 2024-11-15T14:17:07,767 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/37ac3cf93bf14c8886f5c87066cd7038, entries=150, sequenceid=83, filesize=11.7 K 2024-11-15T14:17:07,771 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e34e905c516b4c17b3db18acbd12c2a8 in 975ms, sequenceid=83, compaction requested=false 2024-11-15T14:17:07,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:07,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:07,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-15T14:17:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-15T14:17:07,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-15T14:17:07,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5960 sec 2024-11-15T14:17:07,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.6030 sec 2024-11-15T14:17:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:08,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T14:17:08,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:08,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:08,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:08,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:08,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:08,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:08,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153f92dca82ee842b1ad3300cbcd8c698f_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680227089/Put/seqid=0 2024-11-15T14:17:08,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742399_1575 (size=12154) 2024-11-15T14:17:08,208 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:08,215 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153f92dca82ee842b1ad3300cbcd8c698f_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f92dca82ee842b1ad3300cbcd8c698f_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:08,217 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/835d3f0eaef4465c9ab4ec3652e6b9be, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:08,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/835d3f0eaef4465c9ab4ec3652e6b9be is 175, key is test_row_0/A:col10/1731680227089/Put/seqid=0 2024-11-15T14:17:08,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680288234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680288236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680288237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680288242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680288242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742400_1576 (size=30955) 2024-11-15T14:17:08,265 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/835d3f0eaef4465c9ab4ec3652e6b9be 2024-11-15T14:17:08,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/1844b1375e1f4556878c246895e48255 is 50, key is test_row_0/B:col10/1731680227089/Put/seqid=0 2024-11-15T14:17:08,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742401_1577 (size=12001) 2024-11-15T14:17:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T14:17:08,321 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:08,322 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:17:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-15T14:17:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T14:17:08,323 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:17:08,324 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:17:08,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:17:08,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680288351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680288352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680288352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T14:17:08,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T14:17:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:08,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:08,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:08,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680288561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680288562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680288562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T14:17:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:08,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T14:17:08,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/1844b1375e1f4556878c246895e48255 2024-11-15T14:17:08,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/fae1137e994740b7b02954c4b5dcdeeb is 50, key is test_row_0/C:col10/1731680227089/Put/seqid=0 2024-11-15T14:17:08,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742402_1578 (size=12001) 2024-11-15T14:17:08,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/fae1137e994740b7b02954c4b5dcdeeb 2024-11-15T14:17:08,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/835d3f0eaef4465c9ab4ec3652e6b9be as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/835d3f0eaef4465c9ab4ec3652e6b9be 2024-11-15T14:17:08,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/835d3f0eaef4465c9ab4ec3652e6b9be, entries=150, sequenceid=99, filesize=30.2 K 2024-11-15T14:17:08,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/1844b1375e1f4556878c246895e48255 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1844b1375e1f4556878c246895e48255 2024-11-15T14:17:08,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1844b1375e1f4556878c246895e48255, entries=150, sequenceid=99, filesize=11.7 K 2024-11-15T14:17:08,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/fae1137e994740b7b02954c4b5dcdeeb as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/fae1137e994740b7b02954c4b5dcdeeb 2024-11-15T14:17:08,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/fae1137e994740b7b02954c4b5dcdeeb, entries=150, sequenceid=99, filesize=11.7 K 2024-11-15T14:17:08,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e34e905c516b4c17b3db18acbd12c2a8 in 619ms, sequenceid=99, compaction requested=true 2024-11-15T14:17:08,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:08,765 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:08,766 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:08,766 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/A is initiating minor compaction (all files) 2024-11-15T14:17:08,766 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/A in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:08,766 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/913d1123963d44af9bef5ea03a8e5178, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/94abea38d5a74b3ab9d0a98dd8d7b07d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/835d3f0eaef4465c9ab4ec3652e6b9be] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=90.8 K 2024-11-15T14:17:08,766 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:08,766 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/913d1123963d44af9bef5ea03a8e5178, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/94abea38d5a74b3ab9d0a98dd8d7b07d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/835d3f0eaef4465c9ab4ec3652e6b9be] 2024-11-15T14:17:08,767 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 913d1123963d44af9bef5ea03a8e5178, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731680225229 2024-11-15T14:17:08,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:17:08,767 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94abea38d5a74b3ab9d0a98dd8d7b07d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731680225901 2024-11-15T14:17:08,767 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 835d3f0eaef4465c9ab4ec3652e6b9be, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731680227089 2024-11-15T14:17:08,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:08,768 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:08,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:17:08,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:08,769 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:08,769 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/B is initiating minor compaction (all files) 2024-11-15T14:17:08,769 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/B in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:08,769 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/03e15459b2764977903f1066d6df6dde, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/fb1a32034b3e491bb536d654a33c9494, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1844b1375e1f4556878c246895e48255] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=35.3 K 2024-11-15T14:17:08,770 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 03e15459b2764977903f1066d6df6dde, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731680225229 2024-11-15T14:17:08,772 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting fb1a32034b3e491bb536d654a33c9494, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731680225901 2024-11-15T14:17:08,772 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1844b1375e1f4556878c246895e48255, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731680227089 2024-11-15T14:17:08,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:17:08,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:08,783 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:08,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T14:17:08,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:08,787 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T14:17:08,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:08,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:08,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:08,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:08,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:08,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:08,793 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#B#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:08,793 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/0ce77271703a47fc9d5a5b635932dae3 is 50, key is test_row_0/B:col10/1731680227089/Put/seqid=0 2024-11-15T14:17:08,803 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115b41280f46d1f43b1a1c164b243792812_e34e905c516b4c17b3db18acbd12c2a8 store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:08,806 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115b41280f46d1f43b1a1c164b243792812_e34e905c516b4c17b3db18acbd12c2a8, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:08,806 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b41280f46d1f43b1a1c164b243792812_e34e905c516b4c17b3db18acbd12c2a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:08,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111504abeeff70ac46d88bffb994b8b27294_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680228231/Put/seqid=0 2024-11-15T14:17:08,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742403_1579 (size=12207) 2024-11-15T14:17:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:08,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:08,878 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/0ce77271703a47fc9d5a5b635932dae3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/0ce77271703a47fc9d5a5b635932dae3 2024-11-15T14:17:08,885 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/B of e34e905c516b4c17b3db18acbd12c2a8 into 0ce77271703a47fc9d5a5b635932dae3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:08,885 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:08,885 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/B, priority=13, startTime=1731680228768; duration=0sec 2024-11-15T14:17:08,886 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:08,886 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:B 2024-11-15T14:17:08,886 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:08,887 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:08,887 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/C is initiating minor compaction (all files) 2024-11-15T14:17:08,887 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/C in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:08,887 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/4523ff94c3654eb294fb55cb4cb5ce8b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/37ac3cf93bf14c8886f5c87066cd7038, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/fae1137e994740b7b02954c4b5dcdeeb] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=35.3 K 2024-11-15T14:17:08,887 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4523ff94c3654eb294fb55cb4cb5ce8b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731680225229 2024-11-15T14:17:08,888 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 37ac3cf93bf14c8886f5c87066cd7038, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731680225901 2024-11-15T14:17:08,888 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting fae1137e994740b7b02954c4b5dcdeeb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731680227089 2024-11-15T14:17:08,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742404_1580 (size=4469) 2024-11-15T14:17:08,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742405_1581 (size=12154) 2024-11-15T14:17:08,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:08,896 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#A#compaction#496 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:08,896 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/c3bb533cbb2146139c69817fdd02b3d2 is 175, key is test_row_0/A:col10/1731680227089/Put/seqid=0 2024-11-15T14:17:08,901 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111504abeeff70ac46d88bffb994b8b27294_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111504abeeff70ac46d88bffb994b8b27294_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:08,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/697b14788bb44155bc29b29b1bdeae3b, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:08,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/697b14788bb44155bc29b29b1bdeae3b is 175, key is test_row_0/A:col10/1731680228231/Put/seqid=0 2024-11-15T14:17:08,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742406_1582 (size=31161) 2024-11-15T14:17:08,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680288899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680288901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680288906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:08,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:08,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:08,916 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#C#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:08,917 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/6489190063594281ba5c4490538b30a0 is 50, key is test_row_0/C:col10/1731680227089/Put/seqid=0 2024-11-15T14:17:08,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742407_1583 (size=30955) 2024-11-15T14:17:08,928 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/697b14788bb44155bc29b29b1bdeae3b 2024-11-15T14:17:08,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742408_1584 (size=12207) 2024-11-15T14:17:08,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/f1614da8b57146be9492a5160e224808 is 50, key is test_row_0/B:col10/1731680228231/Put/seqid=0 2024-11-15T14:17:08,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742409_1585 (size=12001) 2024-11-15T14:17:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T14:17:09,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680289012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680289012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:09,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680289022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:09,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680289222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680289222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:09,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680289231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:09,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680289261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:09,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680289262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:09,265 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in e34e905c516b4c17b3db18acbd12c2a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:17:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,265 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in e34e905c516b4c17b3db18acbd12c2a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:17:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:09,307 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/c3bb533cbb2146139c69817fdd02b3d2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/c3bb533cbb2146139c69817fdd02b3d2 2024-11-15T14:17:09,310 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/A of e34e905c516b4c17b3db18acbd12c2a8 into c3bb533cbb2146139c69817fdd02b3d2(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:09,310 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:09,310 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/A, priority=13, startTime=1731680228765; duration=0sec 2024-11-15T14:17:09,310 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:09,310 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:A 2024-11-15T14:17:09,344 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/6489190063594281ba5c4490538b30a0 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/6489190063594281ba5c4490538b30a0 2024-11-15T14:17:09,348 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/C of e34e905c516b4c17b3db18acbd12c2a8 into 6489190063594281ba5c4490538b30a0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:09,348 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:09,348 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/C, priority=13, startTime=1731680228768; duration=0sec 2024-11-15T14:17:09,348 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:09,348 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:C 2024-11-15T14:17:09,352 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/f1614da8b57146be9492a5160e224808 2024-11-15T14:17:09,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/fd4f80e689a04ecb8f1cb86381862496 is 50, key is test_row_0/C:col10/1731680228231/Put/seqid=0 2024-11-15T14:17:09,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742410_1586 (size=12001) 2024-11-15T14:17:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T14:17:09,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680289531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:09,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680289531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:09,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680289541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:09,781 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/fd4f80e689a04ecb8f1cb86381862496 2024-11-15T14:17:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/697b14788bb44155bc29b29b1bdeae3b as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/697b14788bb44155bc29b29b1bdeae3b 2024-11-15T14:17:09,791 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/697b14788bb44155bc29b29b1bdeae3b, entries=150, sequenceid=117, filesize=30.2 K 2024-11-15T14:17:09,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/f1614da8b57146be9492a5160e224808 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/f1614da8b57146be9492a5160e224808 2024-11-15T14:17:09,797 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/f1614da8b57146be9492a5160e224808, entries=150, sequenceid=117, filesize=11.7 K 2024-11-15T14:17:09,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/fd4f80e689a04ecb8f1cb86381862496 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/fd4f80e689a04ecb8f1cb86381862496 2024-11-15T14:17:09,802 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/fd4f80e689a04ecb8f1cb86381862496, entries=150, sequenceid=117, filesize=11.7 K 2024-11-15T14:17:09,804 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e34e905c516b4c17b3db18acbd12c2a8 in 1017ms, sequenceid=117, compaction requested=false 2024-11-15T14:17:09,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:09,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:09,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-15T14:17:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-15T14:17:09,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-15T14:17:09,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4810 sec 2024-11-15T14:17:09,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4860 sec 2024-11-15T14:17:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:10,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T14:17:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:10,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153c443cbdc93b481c881e5b07431f2657_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680228889/Put/seqid=0 2024-11-15T14:17:10,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742411_1587 (size=12254) 2024-11-15T14:17:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680290104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:10,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680290109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:10,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:10,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680290109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:10,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680290212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:10,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680290222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:10,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680290222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:10,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680290421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:10,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680290431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:10,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680290431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T14:17:10,472 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:10,473 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:17:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-15T14:17:10,474 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:17:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T14:17:10,475 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:17:10,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:17:10,490 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:10,493 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153c443cbdc93b481c881e5b07431f2657_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153c443cbdc93b481c881e5b07431f2657_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:10,494 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/57dd081571ac4d09888202c9140fb6c0, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:10,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/57dd081571ac4d09888202c9140fb6c0 is 175, key is test_row_0/A:col10/1731680228889/Put/seqid=0 2024-11-15T14:17:10,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742412_1588 (size=31055) 2024-11-15T14:17:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T14:17:10,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T14:17:10,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:10,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:10,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:10,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:10,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:10,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680290732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:10,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680290742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:10,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680290742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:10,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:10,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T14:17:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:10,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T14:17:10,901 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/57dd081571ac4d09888202c9140fb6c0 2024-11-15T14:17:10,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/5d9109253dc2416c841f9a47376392a1 is 50, key is test_row_0/B:col10/1731680228889/Put/seqid=0 2024-11-15T14:17:10,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742413_1589 (size=12101) 2024-11-15T14:17:10,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/5d9109253dc2416c841f9a47376392a1 2024-11-15T14:17:10,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/c916c4f153714df5977d88642267ccd5 is 50, key is test_row_0/C:col10/1731680228889/Put/seqid=0 2024-11-15T14:17:10,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742414_1590 (size=12101) 2024-11-15T14:17:10,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T14:17:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:10,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:11,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T14:17:11,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:11,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:11,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:11,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:11,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T14:17:11,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T14:17:11,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:11,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:11,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:11,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:11,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:11,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680291242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:11,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680291251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:11,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680291252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:11,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680291282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:11,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680291282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:11,285 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in e34e905c516b4c17b3db18acbd12c2a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:17:11,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:11,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:11,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:11,285 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_0' in e34e905c516b4c17b3db18acbd12c2a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:17:11,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:11,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:11,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:11,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/c916c4f153714df5977d88642267ccd5 2024-11-15T14:17:11,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/57dd081571ac4d09888202c9140fb6c0 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/57dd081571ac4d09888202c9140fb6c0 2024-11-15T14:17:11,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/57dd081571ac4d09888202c9140fb6c0, entries=150, sequenceid=137, filesize=30.3 K 2024-11-15T14:17:11,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/5d9109253dc2416c841f9a47376392a1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/5d9109253dc2416c841f9a47376392a1 2024-11-15T14:17:11,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/5d9109253dc2416c841f9a47376392a1, entries=150, sequenceid=137, filesize=11.8 K 2024-11-15T14:17:11,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/c916c4f153714df5977d88642267ccd5 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/c916c4f153714df5977d88642267ccd5 2024-11-15T14:17:11,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/c916c4f153714df5977d88642267ccd5, entries=150, sequenceid=137, filesize=11.8 K 2024-11-15T14:17:11,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for e34e905c516b4c17b3db18acbd12c2a8 in 1268ms, sequenceid=137, compaction requested=true 2024-11-15T14:17:11,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:11,344 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:11,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:17:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:11,345 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:17:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:17:11,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:11,345 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:11,345 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/A is initiating minor compaction (all files) 2024-11-15T14:17:11,345 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/A in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:11,346 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/c3bb533cbb2146139c69817fdd02b3d2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/697b14788bb44155bc29b29b1bdeae3b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/57dd081571ac4d09888202c9140fb6c0] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=91.0 K 2024-11-15T14:17:11,346 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:11,346 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/c3bb533cbb2146139c69817fdd02b3d2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/697b14788bb44155bc29b29b1bdeae3b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/57dd081571ac4d09888202c9140fb6c0] 2024-11-15T14:17:11,346 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:11,346 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3bb533cbb2146139c69817fdd02b3d2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731680227089 2024-11-15T14:17:11,346 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/B is initiating minor compaction (all files) 2024-11-15T14:17:11,346 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/B in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:11,347 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/0ce77271703a47fc9d5a5b635932dae3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/f1614da8b57146be9492a5160e224808, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/5d9109253dc2416c841f9a47376392a1] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=35.5 K 2024-11-15T14:17:11,347 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ce77271703a47fc9d5a5b635932dae3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731680227089 2024-11-15T14:17:11,347 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 697b14788bb44155bc29b29b1bdeae3b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731680228231 2024-11-15T14:17:11,348 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting f1614da8b57146be9492a5160e224808, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731680228231 2024-11-15T14:17:11,349 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d9109253dc2416c841f9a47376392a1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731680228889 2024-11-15T14:17:11,350 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57dd081571ac4d09888202c9140fb6c0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731680228889 2024-11-15T14:17:11,369 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#B#compaction#505 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:11,369 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/88cbda2006c94b239102fa2fcd88dec6 is 50, key is test_row_0/B:col10/1731680228889/Put/seqid=0 2024-11-15T14:17:11,387 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:11,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T14:17:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:11,391 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T14:17:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:11,397 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111568fdcf41004e4541970ebfd5d678b914_e34e905c516b4c17b3db18acbd12c2a8 store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:11,399 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111568fdcf41004e4541970ebfd5d678b914_e34e905c516b4c17b3db18acbd12c2a8, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:11,400 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111568fdcf41004e4541970ebfd5d678b914_e34e905c516b4c17b3db18acbd12c2a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:11,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742415_1591 (size=12409) 2024-11-15T14:17:11,433 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/88cbda2006c94b239102fa2fcd88dec6 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/88cbda2006c94b239102fa2fcd88dec6 2024-11-15T14:17:11,447 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/B of e34e905c516b4c17b3db18acbd12c2a8 into 88cbda2006c94b239102fa2fcd88dec6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:11,447 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:11,447 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/B, priority=13, startTime=1731680231345; duration=0sec 2024-11-15T14:17:11,447 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:11,447 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:B 2024-11-15T14:17:11,447 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e0ab3eeab1f346f88f56d5df63dcdd09_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680230106/Put/seqid=0 2024-11-15T14:17:11,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742416_1592 (size=4469) 2024-11-15T14:17:11,450 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:11,451 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/C is initiating minor compaction (all files) 2024-11-15T14:17:11,451 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/C in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:11,451 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/6489190063594281ba5c4490538b30a0, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/fd4f80e689a04ecb8f1cb86381862496, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/c916c4f153714df5977d88642267ccd5] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=35.5 K 2024-11-15T14:17:11,451 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 6489190063594281ba5c4490538b30a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731680227089 2024-11-15T14:17:11,451 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting fd4f80e689a04ecb8f1cb86381862496, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731680228231 2024-11-15T14:17:11,452 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c916c4f153714df5977d88642267ccd5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731680228889 2024-11-15T14:17:11,460 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#C#compaction#508 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:11,460 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/9019c331ab634d73a04ce0757da835ca is 50, key is test_row_0/C:col10/1731680228889/Put/seqid=0 2024-11-15T14:17:11,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742418_1594 (size=12409) 2024-11-15T14:17:11,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742417_1593 (size=12304) 2024-11-15T14:17:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T14:17:11,852 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#A#compaction#506 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:11,852 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/2e38cd54965f4d4697bfc77a9357e290 is 175, key is test_row_0/A:col10/1731680228889/Put/seqid=0 2024-11-15T14:17:11,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742419_1595 (size=31363) 2024-11-15T14:17:11,891 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/9019c331ab634d73a04ce0757da835ca as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/9019c331ab634d73a04ce0757da835ca 2024-11-15T14:17:11,895 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/C of e34e905c516b4c17b3db18acbd12c2a8 into 9019c331ab634d73a04ce0757da835ca(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:11,895 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:11,895 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/C, priority=13, startTime=1731680231345; duration=0sec 2024-11-15T14:17:11,895 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:11,895 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:C 2024-11-15T14:17:11,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:11,908 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e0ab3eeab1f346f88f56d5df63dcdd09_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e0ab3eeab1f346f88f56d5df63dcdd09_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:11,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/2610e7db0c99497680122e0f2988f063, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:11,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/2610e7db0c99497680122e0f2988f063 is 175, key is test_row_0/A:col10/1731680230106/Put/seqid=0 2024-11-15T14:17:11,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742420_1596 (size=31105) 2024-11-15T14:17:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:12,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:12,267 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/2e38cd54965f4d4697bfc77a9357e290 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/2e38cd54965f4d4697bfc77a9357e290 2024-11-15T14:17:12,271 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/A of e34e905c516b4c17b3db18acbd12c2a8 into 2e38cd54965f4d4697bfc77a9357e290(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:12,271 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:12,272 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/A, priority=13, startTime=1731680231344; duration=0sec 2024-11-15T14:17:12,272 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:12,272 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:A 2024-11-15T14:17:12,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680292288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680292288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680292288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,313 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/2610e7db0c99497680122e0f2988f063 2024-11-15T14:17:12,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/1ab83572cd23456cbcdc160c48e97083 is 50, key is test_row_0/B:col10/1731680230106/Put/seqid=0 2024-11-15T14:17:12,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742421_1597 (size=12151) 2024-11-15T14:17:12,364 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/1ab83572cd23456cbcdc160c48e97083 2024-11-15T14:17:12,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/1c8b918c870340278bb19e43c7dd546e is 50, key is test_row_0/C:col10/1731680230106/Put/seqid=0 2024-11-15T14:17:12,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742422_1598 (size=12151) 2024-11-15T14:17:12,398 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/1c8b918c870340278bb19e43c7dd546e 2024-11-15T14:17:12,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/2610e7db0c99497680122e0f2988f063 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/2610e7db0c99497680122e0f2988f063 2024-11-15T14:17:12,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680292402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680292402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680292402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,410 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/2610e7db0c99497680122e0f2988f063, entries=150, sequenceid=153, filesize=30.4 K 2024-11-15T14:17:12,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/1ab83572cd23456cbcdc160c48e97083 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1ab83572cd23456cbcdc160c48e97083 2024-11-15T14:17:12,416 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1ab83572cd23456cbcdc160c48e97083, entries=150, sequenceid=153, filesize=11.9 K 2024-11-15T14:17:12,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/1c8b918c870340278bb19e43c7dd546e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/1c8b918c870340278bb19e43c7dd546e 2024-11-15T14:17:12,427 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/1c8b918c870340278bb19e43c7dd546e, entries=150, sequenceid=153, filesize=11.9 K 2024-11-15T14:17:12,428 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e34e905c516b4c17b3db18acbd12c2a8 in 1037ms, sequenceid=153, compaction requested=false 2024-11-15T14:17:12,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:12,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:12,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-15T14:17:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-15T14:17:12,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-15T14:17:12,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9560 sec 2024-11-15T14:17:12,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.9620 sec 2024-11-15T14:17:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:12,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T14:17:12,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:12,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:12,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:12,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:12,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:12,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T14:17:12,621 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:12,622 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:17:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-15T14:17:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T14:17:12,625 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:17:12,626 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:17:12,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:17:12,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115892d0a0bd1424ec190f2ed63483fb12d_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680232614/Put/seqid=0 2024-11-15T14:17:12,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680292637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680292639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680292641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742423_1599 (size=14794) 2024-11-15T14:17:12,689 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:12,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115892d0a0bd1424ec190f2ed63483fb12d_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115892d0a0bd1424ec190f2ed63483fb12d_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:12,697 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/019182046a6744ef9ada0becf88e46af, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:12,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/019182046a6744ef9ada0becf88e46af is 175, key is test_row_0/A:col10/1731680232614/Put/seqid=0 2024-11-15T14:17:12,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742424_1600 (size=39749) 2024-11-15T14:17:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T14:17:12,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680292752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680292752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680292752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T14:17:12,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:12,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:12,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:12,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:12,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:12,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T14:17:12,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:12,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:12,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:12,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T14:17:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680292961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680292961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680292962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:13,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T14:17:13,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:13,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:13,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:13,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:13,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:13,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:13,125 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/019182046a6744ef9ada0becf88e46af 2024-11-15T14:17:13,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/105a74843ed0487bbbc1fa1721ad87ab is 50, key is test_row_0/B:col10/1731680232614/Put/seqid=0 2024-11-15T14:17:13,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742425_1601 (size=12151) 2024-11-15T14:17:13,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/105a74843ed0487bbbc1fa1721ad87ab 2024-11-15T14:17:13,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/387a0de2b19a4d44bcb895b761113a6a is 50, key is test_row_0/C:col10/1731680232614/Put/seqid=0 2024-11-15T14:17:13,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742426_1602 (size=12151) 2024-11-15T14:17:13,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T14:17:13,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:13,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:13,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:13,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:13,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:13,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:13,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T14:17:13,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680293273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:13,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:13,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:13,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680293273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:13,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:13,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680293273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:13,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T14:17:13,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:13,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:13,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:13,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:13,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:13,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T14:17:13,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:13,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:13,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:13,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:13,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/387a0de2b19a4d44bcb895b761113a6a 2024-11-15T14:17:13,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/019182046a6744ef9ada0becf88e46af as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/019182046a6744ef9ada0becf88e46af 2024-11-15T14:17:13,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/019182046a6744ef9ada0becf88e46af, entries=200, sequenceid=174, filesize=38.8 K 2024-11-15T14:17:13,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/105a74843ed0487bbbc1fa1721ad87ab as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/105a74843ed0487bbbc1fa1721ad87ab 2024-11-15T14:17:13,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/105a74843ed0487bbbc1fa1721ad87ab, entries=150, sequenceid=174, filesize=11.9 K 2024-11-15T14:17:13,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/387a0de2b19a4d44bcb895b761113a6a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/387a0de2b19a4d44bcb895b761113a6a 2024-11-15T14:17:13,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/387a0de2b19a4d44bcb895b761113a6a, entries=150, sequenceid=174, filesize=11.9 K 2024-11-15T14:17:13,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e34e905c516b4c17b3db18acbd12c2a8 in 978ms, sequenceid=174, compaction requested=true 2024-11-15T14:17:13,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:13,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:17:13,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:13,594 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:13,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:17:13,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:13,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:17:13,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:17:13,594 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:13,595 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:13,595 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:13,595 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/B is initiating minor compaction (all files) 2024-11-15T14:17:13,595 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/A is initiating minor compaction (all files) 2024-11-15T14:17:13,595 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/A in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:13,595 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/B in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:13,595 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/88cbda2006c94b239102fa2fcd88dec6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1ab83572cd23456cbcdc160c48e97083, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/105a74843ed0487bbbc1fa1721ad87ab] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=35.9 K 2024-11-15T14:17:13,595 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/2e38cd54965f4d4697bfc77a9357e290, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/2610e7db0c99497680122e0f2988f063, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/019182046a6744ef9ada0becf88e46af] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=99.8 K 2024-11-15T14:17:13,595 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:13,595 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/2e38cd54965f4d4697bfc77a9357e290, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/2610e7db0c99497680122e0f2988f063, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/019182046a6744ef9ada0becf88e46af] 2024-11-15T14:17:13,595 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 88cbda2006c94b239102fa2fcd88dec6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731680228889 2024-11-15T14:17:13,596 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e38cd54965f4d4697bfc77a9357e290, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731680228889 2024-11-15T14:17:13,596 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ab83572cd23456cbcdc160c48e97083, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731680230102 2024-11-15T14:17:13,596 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2610e7db0c99497680122e0f2988f063, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731680230102 2024-11-15T14:17:13,596 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 105a74843ed0487bbbc1fa1721ad87ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731680232283 2024-11-15T14:17:13,596 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 019182046a6744ef9ada0becf88e46af, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731680232283 2024-11-15T14:17:13,601 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#B#compaction#514 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:13,602 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/5bfa108129ec4cb2a8a73efbef20da41 is 50, key is test_row_0/B:col10/1731680232614/Put/seqid=0 2024-11-15T14:17:13,603 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:13,605 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115184e758184f5417b9d5c00476a46b21f_e34e905c516b4c17b3db18acbd12c2a8 store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:13,607 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115184e758184f5417b9d5c00476a46b21f_e34e905c516b4c17b3db18acbd12c2a8, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:13,607 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115184e758184f5417b9d5c00476a46b21f_e34e905c516b4c17b3db18acbd12c2a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:13,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742427_1603 (size=12561) 2024-11-15T14:17:13,615 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/5bfa108129ec4cb2a8a73efbef20da41 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/5bfa108129ec4cb2a8a73efbef20da41 2024-11-15T14:17:13,624 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/B of e34e905c516b4c17b3db18acbd12c2a8 into 5bfa108129ec4cb2a8a73efbef20da41(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:13,624 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:13,624 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/B, priority=13, startTime=1731680233594; duration=0sec 2024-11-15T14:17:13,625 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:13,625 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:B 2024-11-15T14:17:13,625 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:13,626 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:13,626 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/C is initiating minor compaction (all files) 2024-11-15T14:17:13,627 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/C in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:13,627 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/9019c331ab634d73a04ce0757da835ca, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/1c8b918c870340278bb19e43c7dd546e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/387a0de2b19a4d44bcb895b761113a6a] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=35.9 K 2024-11-15T14:17:13,627 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9019c331ab634d73a04ce0757da835ca, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731680228889 2024-11-15T14:17:13,627 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c8b918c870340278bb19e43c7dd546e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731680230102 2024-11-15T14:17:13,628 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 387a0de2b19a4d44bcb895b761113a6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731680232283 2024-11-15T14:17:13,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742428_1604 (size=4469) 2024-11-15T14:17:13,632 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#A#compaction#515 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:13,632 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/d3302a6c859a43f6abc8d607b61436b4 is 175, key is test_row_0/A:col10/1731680232614/Put/seqid=0 2024-11-15T14:17:13,649 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#C#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:13,650 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/acf8bc02ec0f40bdafb6d928091901c7 is 50, key is test_row_0/C:col10/1731680232614/Put/seqid=0 2024-11-15T14:17:13,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742430_1606 (size=12561) 2024-11-15T14:17:13,660 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/acf8bc02ec0f40bdafb6d928091901c7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/acf8bc02ec0f40bdafb6d928091901c7 2024-11-15T14:17:13,664 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/C of e34e905c516b4c17b3db18acbd12c2a8 into acf8bc02ec0f40bdafb6d928091901c7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:13,664 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:13,664 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/C, priority=13, startTime=1731680233594; duration=0sec 2024-11-15T14:17:13,664 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:13,664 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:C 2024-11-15T14:17:13,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742429_1605 (size=31515) 2024-11-15T14:17:13,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T14:17:13,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:13,700 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T14:17:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:13,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115712b9a546d72409498d465f966526caa_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680232628/Put/seqid=0 2024-11-15T14:17:13,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742431_1607 (size=12304) 2024-11-15T14:17:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T14:17:13,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:13,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:13,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:13,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680293810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:13,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680293811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:13,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680293813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:13,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680293921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:13,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680293922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:13,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680293932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:13,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:13,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:14,069 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/d3302a6c859a43f6abc8d607b61436b4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/d3302a6c859a43f6abc8d607b61436b4 2024-11-15T14:17:14,073 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/A of e34e905c516b4c17b3db18acbd12c2a8 into d3302a6c859a43f6abc8d607b61436b4(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:14,073 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:14,073 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/A, priority=13, startTime=1731680233594; duration=0sec 2024-11-15T14:17:14,074 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:14,074 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:A 2024-11-15T14:17:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:14,125 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115712b9a546d72409498d465f966526caa_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115712b9a546d72409498d465f966526caa_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:14,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/f44f51829f0543b69fb0c9141e4d6cee, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:14,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/f44f51829f0543b69fb0c9141e4d6cee is 175, key is test_row_0/A:col10/1731680232628/Put/seqid=0 2024-11-15T14:17:14,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742432_1608 (size=31105) 2024-11-15T14:17:14,134 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/f44f51829f0543b69fb0c9141e4d6cee 2024-11-15T14:17:14,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680294132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680294132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:14,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/a9a5e8bae88c484b99b9205e31494d54 is 50, key is test_row_0/B:col10/1731680232628/Put/seqid=0 2024-11-15T14:17:14,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742433_1609 (size=12151) 2024-11-15T14:17:14,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680294142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:14,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680294442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:14,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680294442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:14,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680294451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:14,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:14,545 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/a9a5e8bae88c484b99b9205e31494d54 2024-11-15T14:17:14,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/783455c68a644fda861c0ac05b72a8eb is 50, key is test_row_0/C:col10/1731680232628/Put/seqid=0 2024-11-15T14:17:14,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742434_1610 (size=12151) 2024-11-15T14:17:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T14:17:14,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680294951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:14,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680294952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:14,955 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/783455c68a644fda861c0ac05b72a8eb 2024-11-15T14:17:14,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/f44f51829f0543b69fb0c9141e4d6cee as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f44f51829f0543b69fb0c9141e4d6cee 2024-11-15T14:17:14,960 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f44f51829f0543b69fb0c9141e4d6cee, entries=150, sequenceid=190, filesize=30.4 K 2024-11-15T14:17:14,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/a9a5e8bae88c484b99b9205e31494d54 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/a9a5e8bae88c484b99b9205e31494d54 2024-11-15T14:17:14,963 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/a9a5e8bae88c484b99b9205e31494d54, entries=150, sequenceid=190, filesize=11.9 K 2024-11-15T14:17:14,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/783455c68a644fda861c0ac05b72a8eb as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/783455c68a644fda861c0ac05b72a8eb 2024-11-15T14:17:14,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680294962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:14,966 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/783455c68a644fda861c0ac05b72a8eb, entries=150, sequenceid=190, filesize=11.9 K 2024-11-15T14:17:14,967 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e34e905c516b4c17b3db18acbd12c2a8 in 1267ms, sequenceid=190, compaction requested=false 2024-11-15T14:17:14,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:14,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:14,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-15T14:17:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-15T14:17:14,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-15T14:17:14,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3420 sec 2024-11-15T14:17:14,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.3480 sec 2024-11-15T14:17:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:15,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T14:17:15,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:15,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:15,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:15,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:15,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:15,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:15,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153ad9a003a6db4572ae4457f6cc358054_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680235302/Put/seqid=0 2024-11-15T14:17:15,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742435_1611 (size=14794) 2024-11-15T14:17:15,334 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:15,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153ad9a003a6db4572ae4457f6cc358054_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153ad9a003a6db4572ae4457f6cc358054_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:15,339 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/e22201c04c494cccae7dff33f24285ea, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:15,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/e22201c04c494cccae7dff33f24285ea is 175, key is test_row_0/A:col10/1731680235302/Put/seqid=0 2024-11-15T14:17:15,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680295337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:15,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680295341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:15,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742436_1612 (size=39749) 2024-11-15T14:17:15,354 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/e22201c04c494cccae7dff33f24285ea 2024-11-15T14:17:15,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/cb3aa5c0817c4264862480110ffae695 is 50, key is test_row_0/B:col10/1731680235302/Put/seqid=0 2024-11-15T14:17:15,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742437_1613 (size=12151) 2024-11-15T14:17:15,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680295452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:15,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680295453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:15,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680295662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680295662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:15,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/cb3aa5c0817c4264862480110ffae695 2024-11-15T14:17:15,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/e9a678385079423591f862fa2b4ade4c is 50, key is test_row_0/C:col10/1731680235302/Put/seqid=0 2024-11-15T14:17:15,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742438_1614 (size=12151) 2024-11-15T14:17:15,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680295962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:15,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680295962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:15,963 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in e34e905c516b4c17b3db18acbd12c2a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:17:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:15,964 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in e34e905c516b4c17b3db18acbd12c2a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:17:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:15,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:15,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680295971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680295972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:15,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680295982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:15,987 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in e34e905c516b4c17b3db18acbd12c2a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:17:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:16,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/e9a678385079423591f862fa2b4ade4c 2024-11-15T14:17:16,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/e22201c04c494cccae7dff33f24285ea as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e22201c04c494cccae7dff33f24285ea 2024-11-15T14:17:16,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e22201c04c494cccae7dff33f24285ea, entries=200, sequenceid=210, filesize=38.8 K 2024-11-15T14:17:16,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/cb3aa5c0817c4264862480110ffae695 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/cb3aa5c0817c4264862480110ffae695 2024-11-15T14:17:16,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/cb3aa5c0817c4264862480110ffae695, entries=150, sequenceid=210, filesize=11.9 K 2024-11-15T14:17:16,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/e9a678385079423591f862fa2b4ade4c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e9a678385079423591f862fa2b4ade4c 2024-11-15T14:17:16,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e9a678385079423591f862fa2b4ade4c, entries=150, sequenceid=210, filesize=11.9 K 2024-11-15T14:17:16,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e34e905c516b4c17b3db18acbd12c2a8 in 915ms, sequenceid=210, compaction requested=true 2024-11-15T14:17:16,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:16,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:17:16,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:16,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:17:16,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:17:16,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:17:16,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T14:17:16,219 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:16,219 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:16,226 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:16,226 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/A is initiating minor compaction (all files) 2024-11-15T14:17:16,227 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/A in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:16,227 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/d3302a6c859a43f6abc8d607b61436b4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f44f51829f0543b69fb0c9141e4d6cee, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e22201c04c494cccae7dff33f24285ea] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=100.0 K 2024-11-15T14:17:16,227 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:16,227 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/d3302a6c859a43f6abc8d607b61436b4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f44f51829f0543b69fb0c9141e4d6cee, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e22201c04c494cccae7dff33f24285ea] 2024-11-15T14:17:16,227 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:16,227 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/B is initiating minor compaction (all files) 2024-11-15T14:17:16,227 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/B in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:16,228 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/5bfa108129ec4cb2a8a73efbef20da41, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/a9a5e8bae88c484b99b9205e31494d54, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/cb3aa5c0817c4264862480110ffae695] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=36.0 K 2024-11-15T14:17:16,229 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bfa108129ec4cb2a8a73efbef20da41, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731680232283 2024-11-15T14:17:16,230 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3302a6c859a43f6abc8d607b61436b4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731680232283 2024-11-15T14:17:16,230 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting a9a5e8bae88c484b99b9205e31494d54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731680232628 2024-11-15T14:17:16,230 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting cb3aa5c0817c4264862480110ffae695, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731680233802 2024-11-15T14:17:16,231 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f44f51829f0543b69fb0c9141e4d6cee, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731680232628 2024-11-15T14:17:16,231 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e22201c04c494cccae7dff33f24285ea, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731680233802 2024-11-15T14:17:16,256 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#B#compaction#523 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:16,256 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/c48d90f88b6b435ba3eed7e48ecc03db is 50, key is test_row_0/B:col10/1731680235302/Put/seqid=0 2024-11-15T14:17:16,263 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:16,281 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411157baac92fac12459b9add898f074b9905_e34e905c516b4c17b3db18acbd12c2a8 store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:16,286 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411157baac92fac12459b9add898f074b9905_e34e905c516b4c17b3db18acbd12c2a8, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:16,287 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157baac92fac12459b9add898f074b9905_e34e905c516b4c17b3db18acbd12c2a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:16,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742439_1615 (size=12663) 2024-11-15T14:17:16,298 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/c48d90f88b6b435ba3eed7e48ecc03db as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/c48d90f88b6b435ba3eed7e48ecc03db 2024-11-15T14:17:16,314 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/B of e34e905c516b4c17b3db18acbd12c2a8 into c48d90f88b6b435ba3eed7e48ecc03db(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:16,314 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:16,314 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/B, priority=13, startTime=1731680236219; duration=0sec 2024-11-15T14:17:16,314 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:16,314 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:B 2024-11-15T14:17:16,315 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:16,316 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:16,317 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/C is initiating minor compaction (all files) 2024-11-15T14:17:16,317 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/C in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:16,317 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/acf8bc02ec0f40bdafb6d928091901c7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/783455c68a644fda861c0ac05b72a8eb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e9a678385079423591f862fa2b4ade4c] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=36.0 K 2024-11-15T14:17:16,317 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting acf8bc02ec0f40bdafb6d928091901c7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731680232283 2024-11-15T14:17:16,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742440_1616 (size=4469) 2024-11-15T14:17:16,318 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 783455c68a644fda861c0ac05b72a8eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731680232628 2024-11-15T14:17:16,318 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting e9a678385079423591f862fa2b4ade4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731680233802 2024-11-15T14:17:16,319 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#A#compaction#524 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:16,319 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/ac37307d2f334c999f592f3ae9334b68 is 175, key is test_row_0/A:col10/1731680235302/Put/seqid=0 2024-11-15T14:17:16,341 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#C#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:16,342 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/dd101b606d35435e843fd8e46a143bc3 is 50, key is test_row_0/C:col10/1731680235302/Put/seqid=0 2024-11-15T14:17:16,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742441_1617 (size=31617) 2024-11-15T14:17:16,359 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/ac37307d2f334c999f592f3ae9334b68 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/ac37307d2f334c999f592f3ae9334b68 2024-11-15T14:17:16,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742442_1618 (size=12663) 2024-11-15T14:17:16,364 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/A of e34e905c516b4c17b3db18acbd12c2a8 into ac37307d2f334c999f592f3ae9334b68(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:16,365 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:16,365 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/A, priority=13, startTime=1731680236219; duration=0sec 2024-11-15T14:17:16,367 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:16,367 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:A 2024-11-15T14:17:16,367 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/dd101b606d35435e843fd8e46a143bc3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/dd101b606d35435e843fd8e46a143bc3 2024-11-15T14:17:16,376 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/C of e34e905c516b4c17b3db18acbd12c2a8 into dd101b606d35435e843fd8e46a143bc3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:16,377 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:16,377 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/C, priority=13, startTime=1731680236219; duration=0sec 2024-11-15T14:17:16,377 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:16,377 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:C 2024-11-15T14:17:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:16,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T14:17:16,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:16,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:16,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:16,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:16,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:16,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:16,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d1d8126da8de47aa99ecebfe074a0a42_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680236484/Put/seqid=0 2024-11-15T14:17:16,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742443_1619 (size=12304) 2024-11-15T14:17:16,519 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:16,531 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d1d8126da8de47aa99ecebfe074a0a42_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d1d8126da8de47aa99ecebfe074a0a42_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:16,534 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/e0fe0aedcee34489b7cb0f5e4f6c8762, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:16,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/e0fe0aedcee34489b7cb0f5e4f6c8762 is 175, key is test_row_0/A:col10/1731680236484/Put/seqid=0 2024-11-15T14:17:16,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742444_1620 (size=31105) 2024-11-15T14:17:16,578 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/e0fe0aedcee34489b7cb0f5e4f6c8762 2024-11-15T14:17:16,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680296584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:16,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680296583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:16,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/1ae2163ba0364d1694ab89368b71f9a7 is 50, key is test_row_0/B:col10/1731680236484/Put/seqid=0 2024-11-15T14:17:16,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742445_1621 (size=12151) 2024-11-15T14:17:16,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680296693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:16,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680296698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T14:17:16,793 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:16,795 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:17:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-15T14:17:16,804 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:17:16,805 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:17:16,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:17:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T14:17:16,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680296902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:16,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:16,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:16,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T14:17:16,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680296911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:16,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:16,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:16,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:16,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T14:17:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:16,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:16,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/1ae2163ba0364d1694ab89368b71f9a7 2024-11-15T14:17:17,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/60e06f11c6e9414ea86dc3a4c147b95c is 50, key is test_row_0/C:col10/1731680236484/Put/seqid=0 2024-11-15T14:17:17,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742446_1622 (size=12151) 2024-11-15T14:17:17,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/60e06f11c6e9414ea86dc3a4c147b95c 2024-11-15T14:17:17,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T14:17:17,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:17,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:17,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:17,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/e0fe0aedcee34489b7cb0f5e4f6c8762 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e0fe0aedcee34489b7cb0f5e4f6c8762 2024-11-15T14:17:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T14:17:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e0fe0aedcee34489b7cb0f5e4f6c8762, entries=150, sequenceid=228, filesize=30.4 K 2024-11-15T14:17:17,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/1ae2163ba0364d1694ab89368b71f9a7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1ae2163ba0364d1694ab89368b71f9a7 2024-11-15T14:17:17,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1ae2163ba0364d1694ab89368b71f9a7, entries=150, sequenceid=228, filesize=11.9 K 2024-11-15T14:17:17,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/60e06f11c6e9414ea86dc3a4c147b95c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/60e06f11c6e9414ea86dc3a4c147b95c 2024-11-15T14:17:17,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/60e06f11c6e9414ea86dc3a4c147b95c, entries=150, sequenceid=228, filesize=11.9 K 2024-11-15T14:17:17,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e34e905c516b4c17b3db18acbd12c2a8 in 662ms, sequenceid=228, compaction requested=false 2024-11-15T14:17:17,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:17,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T14:17:17,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:17,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:17,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:17,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:17,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:17,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:17,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115af003f1ace9047f99b96e531565b6cca_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680236560/Put/seqid=0 2024-11-15T14:17:17,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742447_1623 (size=14794) 2024-11-15T14:17:17,243 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:17,246 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115af003f1ace9047f99b96e531565b6cca_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115af003f1ace9047f99b96e531565b6cca_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:17,247 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/8b262803a6e64defb24c5cd86c98815d, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:17,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/8b262803a6e64defb24c5cd86c98815d is 175, key is test_row_0/A:col10/1731680236560/Put/seqid=0 2024-11-15T14:17:17,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680297250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:17,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680297251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:17,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:17,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742448_1624 (size=39749) 2024-11-15T14:17:17,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T14:17:17,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:17,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:17,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:17,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680297361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:17,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680297361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:17,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T14:17:17,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:17,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:17,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:17,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T14:17:17,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680297572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:17,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:17,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680297572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:17,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T14:17:17,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:17,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:17,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:17,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,660 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/8b262803a6e64defb24c5cd86c98815d 2024-11-15T14:17:17,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/75413411c5694dfcb4ae198f71eb89fb is 50, key is test_row_0/B:col10/1731680236560/Put/seqid=0 2024-11-15T14:17:17,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742449_1625 (size=12151) 2024-11-15T14:17:17,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T14:17:17,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:17,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:17,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:17,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680297882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680297883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:17,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T14:17:17,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:17,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:17,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:17,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T14:17:17,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680297972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:17,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680297972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:17,976 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in e34e905c516b4c17b3db18acbd12c2a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:17:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:17,976 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in e34e905c516b4c17b3db18acbd12c2a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:17:17,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:17,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:18,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680298001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:18,006 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in e34e905c516b4c17b3db18acbd12c2a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:17:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:18,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T14:17:18,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:18,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:18,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:18,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:18,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:18,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/75413411c5694dfcb4ae198f71eb89fb 2024-11-15T14:17:18,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/cbcd464fb72a4e328dc4ce918db0de9e is 50, key is test_row_0/C:col10/1731680236560/Put/seqid=0 2024-11-15T14:17:18,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742450_1626 (size=12151) 2024-11-15T14:17:18,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/cbcd464fb72a4e328dc4ce918db0de9e 2024-11-15T14:17:18,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/8b262803a6e64defb24c5cd86c98815d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/8b262803a6e64defb24c5cd86c98815d 2024-11-15T14:17:18,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/8b262803a6e64defb24c5cd86c98815d, entries=200, sequenceid=246, filesize=38.8 K 2024-11-15T14:17:18,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/75413411c5694dfcb4ae198f71eb89fb as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/75413411c5694dfcb4ae198f71eb89fb 2024-11-15T14:17:18,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/75413411c5694dfcb4ae198f71eb89fb, entries=150, sequenceid=246, filesize=11.9 K 2024-11-15T14:17:18,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/cbcd464fb72a4e328dc4ce918db0de9e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/cbcd464fb72a4e328dc4ce918db0de9e 2024-11-15T14:17:18,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/cbcd464fb72a4e328dc4ce918db0de9e, entries=150, sequenceid=246, filesize=11.9 K 2024-11-15T14:17:18,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e34e905c516b4c17b3db18acbd12c2a8 in 928ms, sequenceid=246, compaction requested=true 2024-11-15T14:17:18,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:18,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:17:18,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:18,144 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:18,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:17:18,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:18,144 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:18,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:17:18,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:18,145 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:18,145 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:18,145 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/A is initiating minor compaction (all files) 2024-11-15T14:17:18,145 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/B is initiating minor compaction (all files) 2024-11-15T14:17:18,145 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/A in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:18,145 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/B in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:18,145 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/c48d90f88b6b435ba3eed7e48ecc03db, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1ae2163ba0364d1694ab89368b71f9a7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/75413411c5694dfcb4ae198f71eb89fb] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=36.1 K 2024-11-15T14:17:18,145 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/ac37307d2f334c999f592f3ae9334b68, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e0fe0aedcee34489b7cb0f5e4f6c8762, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/8b262803a6e64defb24c5cd86c98815d] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=100.1 K 2024-11-15T14:17:18,145 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:18,145 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/ac37307d2f334c999f592f3ae9334b68, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e0fe0aedcee34489b7cb0f5e4f6c8762, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/8b262803a6e64defb24c5cd86c98815d] 2024-11-15T14:17:18,145 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c48d90f88b6b435ba3eed7e48ecc03db, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731680233802 2024-11-15T14:17:18,146 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac37307d2f334c999f592f3ae9334b68, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731680233802 2024-11-15T14:17:18,146 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ae2163ba0364d1694ab89368b71f9a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731680235340 2024-11-15T14:17:18,147 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0fe0aedcee34489b7cb0f5e4f6c8762, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731680235340 2024-11-15T14:17:18,147 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b262803a6e64defb24c5cd86c98815d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731680236554 2024-11-15T14:17:18,147 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 75413411c5694dfcb4ae198f71eb89fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731680236554 2024-11-15T14:17:18,175 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:18,189 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#B#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:18,189 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/966c0ef865934f1882c860622c9379d1 is 50, key is test_row_0/B:col10/1731680236560/Put/seqid=0 2024-11-15T14:17:18,191 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111538347cd6d24648d7b2c6c4bb5f6ff9ba_e34e905c516b4c17b3db18acbd12c2a8 store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:18,194 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111538347cd6d24648d7b2c6c4bb5f6ff9ba_e34e905c516b4c17b3db18acbd12c2a8, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:18,194 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111538347cd6d24648d7b2c6c4bb5f6ff9ba_e34e905c516b4c17b3db18acbd12c2a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:18,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T14:17:18,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:18,207 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T14:17:18,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:18,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:18,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:18,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:18,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:18,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:18,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742451_1627 (size=12765) 2024-11-15T14:17:18,224 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/966c0ef865934f1882c860622c9379d1 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/966c0ef865934f1882c860622c9379d1 2024-11-15T14:17:18,229 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/B of e34e905c516b4c17b3db18acbd12c2a8 into 966c0ef865934f1882c860622c9379d1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:18,230 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:18,230 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/B, priority=13, startTime=1731680238144; duration=0sec 2024-11-15T14:17:18,230 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:18,230 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:B 2024-11-15T14:17:18,230 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:18,235 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:18,235 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/C is initiating minor compaction (all files) 2024-11-15T14:17:18,235 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/C in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:18,236 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/dd101b606d35435e843fd8e46a143bc3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/60e06f11c6e9414ea86dc3a4c147b95c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/cbcd464fb72a4e328dc4ce918db0de9e] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=36.1 K 2024-11-15T14:17:18,236 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting dd101b606d35435e843fd8e46a143bc3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731680233802 2024-11-15T14:17:18,236 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 60e06f11c6e9414ea86dc3a4c147b95c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731680235340 2024-11-15T14:17:18,237 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting cbcd464fb72a4e328dc4ce918db0de9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731680236554 2024-11-15T14:17:18,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bd9e5ec7f3904b40a31bee0f4e3d13bb_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680237250/Put/seqid=0 2024-11-15T14:17:18,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742452_1628 (size=4469) 2024-11-15T14:17:18,248 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#C#compaction#535 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:18,249 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/0333c0661d8d4410a8514cedcdd7cdc8 is 50, key is test_row_0/C:col10/1731680236560/Put/seqid=0 2024-11-15T14:17:18,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742453_1629 (size=12404) 2024-11-15T14:17:18,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742454_1630 (size=12765) 2024-11-15T14:17:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:18,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:18,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680298426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:18,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680298427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:18,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680298531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:18,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680298542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:18,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:18,645 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#A#compaction#532 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:18,645 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/c24596ff3577465cb825d2e49777b621 is 175, key is test_row_0/A:col10/1731680236560/Put/seqid=0 2024-11-15T14:17:18,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742455_1631 (size=31719) 2024-11-15T14:17:18,651 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/c24596ff3577465cb825d2e49777b621 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/c24596ff3577465cb825d2e49777b621 2024-11-15T14:17:18,655 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/A of e34e905c516b4c17b3db18acbd12c2a8 into c24596ff3577465cb825d2e49777b621(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:18,655 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:18,655 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/A, priority=13, startTime=1731680238144; duration=0sec 2024-11-15T14:17:18,655 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:18,655 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:A 2024-11-15T14:17:18,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:18,671 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bd9e5ec7f3904b40a31bee0f4e3d13bb_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bd9e5ec7f3904b40a31bee0f4e3d13bb_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:18,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/f304b802c4034f12b127f03148270789, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:18,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/f304b802c4034f12b127f03148270789 is 175, key is test_row_0/A:col10/1731680237250/Put/seqid=0 2024-11-15T14:17:18,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742456_1632 (size=31205) 2024-11-15T14:17:18,696 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/0333c0661d8d4410a8514cedcdd7cdc8 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/0333c0661d8d4410a8514cedcdd7cdc8 2024-11-15T14:17:18,699 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/C of e34e905c516b4c17b3db18acbd12c2a8 into 0333c0661d8d4410a8514cedcdd7cdc8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:18,699 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:18,699 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/C, priority=13, startTime=1731680238144; duration=0sec 2024-11-15T14:17:18,699 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:18,699 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:C 2024-11-15T14:17:18,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680298741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:18,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680298751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T14:17:19,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680299052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:19,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680299062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:19,076 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/f304b802c4034f12b127f03148270789 2024-11-15T14:17:19,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/02eb1b26691b4958849946eb6a0d1694 is 50, key is test_row_0/B:col10/1731680237250/Put/seqid=0 2024-11-15T14:17:19,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742457_1633 (size=12251) 2024-11-15T14:17:19,485 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/02eb1b26691b4958849946eb6a0d1694 2024-11-15T14:17:19,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/edd7ae9680b04ff695de0fc6012114a7 is 50, key is test_row_0/C:col10/1731680237250/Put/seqid=0 2024-11-15T14:17:19,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742458_1634 (size=12251) 2024-11-15T14:17:19,500 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/edd7ae9680b04ff695de0fc6012114a7 2024-11-15T14:17:19,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/f304b802c4034f12b127f03148270789 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f304b802c4034f12b127f03148270789 2024-11-15T14:17:19,513 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f304b802c4034f12b127f03148270789, entries=150, sequenceid=260, filesize=30.5 K 2024-11-15T14:17:19,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/02eb1b26691b4958849946eb6a0d1694 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/02eb1b26691b4958849946eb6a0d1694 2024-11-15T14:17:19,519 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/02eb1b26691b4958849946eb6a0d1694, entries=150, sequenceid=260, filesize=12.0 K 2024-11-15T14:17:19,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/edd7ae9680b04ff695de0fc6012114a7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/edd7ae9680b04ff695de0fc6012114a7 2024-11-15T14:17:19,526 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/edd7ae9680b04ff695de0fc6012114a7, entries=150, sequenceid=260, filesize=12.0 K 2024-11-15T14:17:19,527 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e34e905c516b4c17b3db18acbd12c2a8 in 1320ms, sequenceid=260, compaction requested=false 2024-11-15T14:17:19,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:19,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:19,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-15T14:17:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-15T14:17:19,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-15T14:17:19,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7250 sec 2024-11-15T14:17:19,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.7370 sec 2024-11-15T14:17:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:19,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T14:17:19,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:19,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:19,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:19,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:19,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:19,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:19,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e660fc6e42d143d49a3a5794df49689e_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680239561/Put/seqid=0 2024-11-15T14:17:19,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742459_1635 (size=14994) 2024-11-15T14:17:19,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680299589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:19,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680299590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:19,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680299703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:19,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:19,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680299703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:19,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680299912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:19,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680299912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:19,972 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:19,976 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e660fc6e42d143d49a3a5794df49689e_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e660fc6e42d143d49a3a5794df49689e_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:19,977 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/b4b0bd76f2284738b6c3f6135154f88d, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:19,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/b4b0bd76f2284738b6c3f6135154f88d is 175, key is test_row_0/A:col10/1731680239561/Put/seqid=0 2024-11-15T14:17:19,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742460_1636 (size=39949) 2024-11-15T14:17:19,984 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/b4b0bd76f2284738b6c3f6135154f88d 2024-11-15T14:17:19,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/29c8529d72794ad6bf873c877c1c2f5e is 50, key is test_row_0/B:col10/1731680239561/Put/seqid=0 2024-11-15T14:17:19,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742461_1637 (size=12301) 2024-11-15T14:17:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680300222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680300222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:20,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/29c8529d72794ad6bf873c877c1c2f5e 2024-11-15T14:17:20,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/b3f8f061a609432e946a6c647543209d is 50, key is test_row_0/C:col10/1731680239561/Put/seqid=0 2024-11-15T14:17:20,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742462_1638 (size=12301) 2024-11-15T14:17:20,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680300732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:20,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680300732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:20,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/b3f8f061a609432e946a6c647543209d 2024-11-15T14:17:20,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/b4b0bd76f2284738b6c3f6135154f88d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/b4b0bd76f2284738b6c3f6135154f88d 2024-11-15T14:17:20,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/b4b0bd76f2284738b6c3f6135154f88d, entries=200, sequenceid=282, filesize=39.0 K 2024-11-15T14:17:20,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/29c8529d72794ad6bf873c877c1c2f5e as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/29c8529d72794ad6bf873c877c1c2f5e 2024-11-15T14:17:20,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/29c8529d72794ad6bf873c877c1c2f5e, entries=150, sequenceid=282, filesize=12.0 K 2024-11-15T14:17:20,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/b3f8f061a609432e946a6c647543209d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/b3f8f061a609432e946a6c647543209d 2024-11-15T14:17:20,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/b3f8f061a609432e946a6c647543209d, entries=150, sequenceid=282, filesize=12.0 K 2024-11-15T14:17:20,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e34e905c516b4c17b3db18acbd12c2a8 in 1268ms, sequenceid=282, compaction requested=true 2024-11-15T14:17:20,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:20,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:17:20,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:20,830 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:20,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:17:20,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:20,830 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:20,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:17:20,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:20,832 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:20,832 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:20,832 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/B is initiating minor compaction (all files) 2024-11-15T14:17:20,832 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/A is initiating minor compaction (all files) 2024-11-15T14:17:20,832 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/A in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:20,832 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/B in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:20,832 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/c24596ff3577465cb825d2e49777b621, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f304b802c4034f12b127f03148270789, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/b4b0bd76f2284738b6c3f6135154f88d] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=100.5 K 2024-11-15T14:17:20,832 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/966c0ef865934f1882c860622c9379d1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/02eb1b26691b4958849946eb6a0d1694, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/29c8529d72794ad6bf873c877c1c2f5e] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=36.4 K 2024-11-15T14:17:20,832 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:20,832 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/c24596ff3577465cb825d2e49777b621, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f304b802c4034f12b127f03148270789, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/b4b0bd76f2284738b6c3f6135154f88d] 2024-11-15T14:17:20,832 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 966c0ef865934f1882c860622c9379d1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731680236554 2024-11-15T14:17:20,832 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c24596ff3577465cb825d2e49777b621, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731680236554 2024-11-15T14:17:20,833 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 02eb1b26691b4958849946eb6a0d1694, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731680237223 2024-11-15T14:17:20,833 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f304b802c4034f12b127f03148270789, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731680237223 2024-11-15T14:17:20,833 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 29c8529d72794ad6bf873c877c1c2f5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731680238420 2024-11-15T14:17:20,833 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4b0bd76f2284738b6c3f6135154f88d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731680238420 2024-11-15T14:17:20,837 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:20,838 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#B#compaction#541 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:20,838 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/cdb618a2bbf945e1bbeabc6578be4102 is 50, key is test_row_0/B:col10/1731680239561/Put/seqid=0 2024-11-15T14:17:20,838 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115adc8d898ae6e43069bf05242b5a335ac_e34e905c516b4c17b3db18acbd12c2a8 store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:20,840 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115adc8d898ae6e43069bf05242b5a335ac_e34e905c516b4c17b3db18acbd12c2a8, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:20,840 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115adc8d898ae6e43069bf05242b5a335ac_e34e905c516b4c17b3db18acbd12c2a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:20,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742463_1639 (size=13017) 2024-11-15T14:17:20,851 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/cdb618a2bbf945e1bbeabc6578be4102 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/cdb618a2bbf945e1bbeabc6578be4102 2024-11-15T14:17:20,855 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/B of e34e905c516b4c17b3db18acbd12c2a8 into cdb618a2bbf945e1bbeabc6578be4102(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:20,855 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:20,855 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/B, priority=13, startTime=1731680240830; duration=0sec 2024-11-15T14:17:20,856 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:20,856 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:B 2024-11-15T14:17:20,856 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:20,856 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:20,856 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/C is initiating minor compaction (all files) 2024-11-15T14:17:20,856 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/C in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:20,857 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/0333c0661d8d4410a8514cedcdd7cdc8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/edd7ae9680b04ff695de0fc6012114a7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/b3f8f061a609432e946a6c647543209d] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=36.4 K 2024-11-15T14:17:20,857 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0333c0661d8d4410a8514cedcdd7cdc8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731680236554 2024-11-15T14:17:20,857 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting edd7ae9680b04ff695de0fc6012114a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731680237223 2024-11-15T14:17:20,857 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting b3f8f061a609432e946a6c647543209d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731680238420 2024-11-15T14:17:20,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742464_1640 (size=4469) 2024-11-15T14:17:20,864 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#C#compaction#543 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:20,865 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/420ab49b4b8d47838f3ac11c3f6a0fff is 50, key is test_row_0/C:col10/1731680239561/Put/seqid=0 2024-11-15T14:17:20,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742465_1641 (size=13017) 2024-11-15T14:17:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T14:17:20,971 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:20,972 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:17:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-15T14:17:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T14:17:20,974 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:17:20,975 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:17:20,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:17:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T14:17:21,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T14:17:21,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:21,127 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T14:17:21,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:21,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:21,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f3dc681f0e834fc68b30e8a2665ee309_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680239588/Put/seqid=0 2024-11-15T14:17:21,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742466_1642 (size=12454) 2024-11-15T14:17:21,261 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#A#compaction#542 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:21,261 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/dd9b8d17a902484f9c806d17104ff63d is 175, key is test_row_0/A:col10/1731680239561/Put/seqid=0 2024-11-15T14:17:21,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742467_1643 (size=31971) 2024-11-15T14:17:21,273 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/420ab49b4b8d47838f3ac11c3f6a0fff as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/420ab49b4b8d47838f3ac11c3f6a0fff 2024-11-15T14:17:21,278 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/C of e34e905c516b4c17b3db18acbd12c2a8 into 420ab49b4b8d47838f3ac11c3f6a0fff(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:21,278 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:21,278 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/C, priority=13, startTime=1731680240830; duration=0sec 2024-11-15T14:17:21,278 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:21,278 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:C 2024-11-15T14:17:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T14:17:21,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:21,546 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f3dc681f0e834fc68b30e8a2665ee309_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f3dc681f0e834fc68b30e8a2665ee309_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:21,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/e084c48558db47c18aaead21696782d5, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:21,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/e084c48558db47c18aaead21696782d5 is 175, key is test_row_0/A:col10/1731680239588/Put/seqid=0 2024-11-15T14:17:21,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742468_1644 (size=31255) 2024-11-15T14:17:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T14:17:21,671 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/dd9b8d17a902484f9c806d17104ff63d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/dd9b8d17a902484f9c806d17104ff63d 2024-11-15T14:17:21,678 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/A of e34e905c516b4c17b3db18acbd12c2a8 into dd9b8d17a902484f9c806d17104ff63d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:21,678 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:21,678 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/A, priority=13, startTime=1731680240830; duration=0sec 2024-11-15T14:17:21,678 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:21,678 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:A 2024-11-15T14:17:21,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:21,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680301782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:21,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:21,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:21,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:21,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680301786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:21,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680301892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:21,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680301902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:21,952 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/e084c48558db47c18aaead21696782d5 2024-11-15T14:17:21,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/c1e2233044f442328a9a4c436d44c2ca is 50, key is test_row_0/B:col10/1731680239588/Put/seqid=0 2024-11-15T14:17:21,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742469_1645 (size=12301) 2024-11-15T14:17:22,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37644 deadline: 1731680302002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:22,005 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in e34e905c516b4c17b3db18acbd12c2a8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:17:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:22,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37652 deadline: 1731680302012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:22,016 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_1' in e34e905c516b4c17b3db18acbd12c2a8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:17:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:22,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37638 deadline: 1731680302052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:22,056 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 26754e923c4a,40177,1731680077530 for 'test_row_2' in e34e905c516b4c17b3db18acbd12c2a8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-15T14:17:22,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:22,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680302102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T14:17:22,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680302111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:22,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:22,380 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/c1e2233044f442328a9a4c436d44c2ca 2024-11-15T14:17:22,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/141d194711f441f788bea572ce6b868f is 50, key is test_row_0/C:col10/1731680239588/Put/seqid=0 2024-11-15T14:17:22,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742470_1646 (size=12301) 2024-11-15T14:17:22,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680302412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:22,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680302422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:22,789 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/141d194711f441f788bea572ce6b868f 2024-11-15T14:17:22,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/e084c48558db47c18aaead21696782d5 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e084c48558db47c18aaead21696782d5 2024-11-15T14:17:22,795 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e084c48558db47c18aaead21696782d5, entries=150, sequenceid=296, filesize=30.5 K 2024-11-15T14:17:22,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/c1e2233044f442328a9a4c436d44c2ca as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/c1e2233044f442328a9a4c436d44c2ca 2024-11-15T14:17:22,798 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/c1e2233044f442328a9a4c436d44c2ca, entries=150, sequenceid=296, filesize=12.0 K 2024-11-15T14:17:22,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/141d194711f441f788bea572ce6b868f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/141d194711f441f788bea572ce6b868f 2024-11-15T14:17:22,801 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/141d194711f441f788bea572ce6b868f, entries=150, sequenceid=296, filesize=12.0 K 2024-11-15T14:17:22,802 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e34e905c516b4c17b3db18acbd12c2a8 in 1675ms, sequenceid=296, compaction requested=false 2024-11-15T14:17:22,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:22,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:22,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-15T14:17:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-15T14:17:22,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-15T14:17:22,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8290 sec 2024-11-15T14:17:22,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8340 sec 2024-11-15T14:17:22,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T14:17:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:22,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:22,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:17:22,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:22,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:17:22,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:22,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:17:22,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b694a30fcff04d7a8c2d4b64ab45287c_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680242922/Put/seqid=0 2024-11-15T14:17:22,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680302963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:22,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680302964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:23,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680303071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:23,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680303071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:23,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742471_1647 (size=17534) 2024-11-15T14:17:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T14:17:23,122 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:23,123 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T14:17:23,244 INFO [AsyncFSWAL-0-hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData-prefix:26754e923c4a,35807,1731680075796 {}] wal.AbstractFSWAL(1368): Slow sync cost: 120 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45147,DS-23ad49e8-4c57-496c-874d-0b5377ea84ae,DISK]] 2024-11-15T14:17:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-15T14:17:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T14:17:23,246 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T14:17:23,246 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T14:17:23,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T14:17:23,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680303282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680303282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T14:17:23,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T14:17:23,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:23,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:23,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:23,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:23,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T14:17:23,476 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:23,479 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b694a30fcff04d7a8c2d4b64ab45287c_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b694a30fcff04d7a8c2d4b64ab45287c_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:23,480 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/638583b87f5743bbadf05ece9c006060, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:23,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/638583b87f5743bbadf05ece9c006060 is 175, key is test_row_0/A:col10/1731680242922/Put/seqid=0 2024-11-15T14:17:23,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742472_1648 (size=48639) 2024-11-15T14:17:23,483 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/638583b87f5743bbadf05ece9c006060 2024-11-15T14:17:23,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/315336fb372946bd8189955aa171ffa3 is 50, key is test_row_0/B:col10/1731680242922/Put/seqid=0 2024-11-15T14:17:23,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742473_1649 (size=12301) 2024-11-15T14:17:23,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/315336fb372946bd8189955aa171ffa3 2024-11-15T14:17:23,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/e36e5fd9d6204d69bfa13374ade3a6ba is 50, key is test_row_0/C:col10/1731680242922/Put/seqid=0 2024-11-15T14:17:23,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742474_1650 (size=12301) 2024-11-15T14:17:23,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/e36e5fd9d6204d69bfa13374ade3a6ba 2024-11-15T14:17:23,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/638583b87f5743bbadf05ece9c006060 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/638583b87f5743bbadf05ece9c006060 2024-11-15T14:17:23,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/638583b87f5743bbadf05ece9c006060, entries=250, sequenceid=319, filesize=47.5 K 2024-11-15T14:17:23,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/315336fb372946bd8189955aa171ffa3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/315336fb372946bd8189955aa171ffa3 2024-11-15T14:17:23,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/315336fb372946bd8189955aa171ffa3, entries=150, sequenceid=319, filesize=12.0 K 2024-11-15T14:17:23,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/e36e5fd9d6204d69bfa13374ade3a6ba as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e36e5fd9d6204d69bfa13374ade3a6ba 2024-11-15T14:17:23,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e36e5fd9d6204d69bfa13374ade3a6ba, entries=150, sequenceid=319, filesize=12.0 K 2024-11-15T14:17:23,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for e34e905c516b4c17b3db18acbd12c2a8 in 602ms, sequenceid=319, compaction requested=true 2024-11-15T14:17:23,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:23,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:17:23,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:23,542 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:23,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:17:23,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:23,542 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:23,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:17:23,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:23,543 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:23,543 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:23,543 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/A is initiating minor compaction (all files) 2024-11-15T14:17:23,543 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/B is initiating minor compaction (all files) 2024-11-15T14:17:23,543 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/B in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:23,543 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/A in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:23,543 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/dd9b8d17a902484f9c806d17104ff63d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e084c48558db47c18aaead21696782d5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/638583b87f5743bbadf05ece9c006060] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=109.2 K 2024-11-15T14:17:23,543 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/cdb618a2bbf945e1bbeabc6578be4102, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/c1e2233044f442328a9a4c436d44c2ca, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/315336fb372946bd8189955aa171ffa3] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=36.7 K 2024-11-15T14:17:23,543 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:23,543 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/dd9b8d17a902484f9c806d17104ff63d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e084c48558db47c18aaead21696782d5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/638583b87f5743bbadf05ece9c006060] 2024-11-15T14:17:23,543 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting cdb618a2bbf945e1bbeabc6578be4102, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731680238420 2024-11-15T14:17:23,544 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd9b8d17a902484f9c806d17104ff63d, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731680238420 2024-11-15T14:17:23,544 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting c1e2233044f442328a9a4c436d44c2ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731680239573 2024-11-15T14:17:23,544 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e084c48558db47c18aaead21696782d5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731680239573 2024-11-15T14:17:23,544 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 315336fb372946bd8189955aa171ffa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731680241776 2024-11-15T14:17:23,544 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 638583b87f5743bbadf05ece9c006060, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731680241764 2024-11-15T14:17:23,550 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#B#compaction#550 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:23,550 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/0991f76acf964715be726236c2d362f0 is 50, key is test_row_0/B:col10/1731680242922/Put/seqid=0 2024-11-15T14:17:23,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T14:17:23,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:23,551 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T14:17:23,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:23,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:23,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:23,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:23,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:23,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:23,553 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:23,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742475_1651 (size=13119) 2024-11-15T14:17:23,558 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115f6c1e014c90f4da49643ffc676c5867d_e34e905c516b4c17b3db18acbd12c2a8 store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:23,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f60b9a9513e14db3b4e4cb05d050912c_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680242962/Put/seqid=0 2024-11-15T14:17:23,560 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115f6c1e014c90f4da49643ffc676c5867d_e34e905c516b4c17b3db18acbd12c2a8, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:23,560 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f6c1e014c90f4da49643ffc676c5867d_e34e905c516b4c17b3db18acbd12c2a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T14:17:23,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742477_1653 (size=4469) 2024-11-15T14:17:23,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742476_1652 (size=12454) 2024-11-15T14:17:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:23,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. as already flushing 2024-11-15T14:17:23,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680243618, max timestamp=1731680243637Num uniques 859; 2024-11-15T14:17:23,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:17:23,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680243618, max timestamp=1731680243637Num uniques 859; 2024-11-15T14:17:23,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:17:23,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731680243618, max timestamp=1731680243637Num uniques 859; 2024-11-15T14:17:23,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/26754e923c4a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T14:17:23,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680303681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:23,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680303687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:23,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680303791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:23,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680303802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T14:17:23,959 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/0991f76acf964715be726236c2d362f0 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/0991f76acf964715be726236c2d362f0 2024-11-15T14:17:23,963 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/B of e34e905c516b4c17b3db18acbd12c2a8 into 0991f76acf964715be726236c2d362f0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:23,963 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:23,963 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/B, priority=13, startTime=1731680243542; duration=0sec 2024-11-15T14:17:23,963 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:23,963 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:B 2024-11-15T14:17:23,963 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:23,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:23,964 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:23,964 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/C is initiating minor compaction (all files) 2024-11-15T14:17:23,964 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/C in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:23,964 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/420ab49b4b8d47838f3ac11c3f6a0fff, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/141d194711f441f788bea572ce6b868f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e36e5fd9d6204d69bfa13374ade3a6ba] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=36.7 K 2024-11-15T14:17:23,964 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#A#compaction#551 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:23,965 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 420ab49b4b8d47838f3ac11c3f6a0fff, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731680238420 2024-11-15T14:17:23,965 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/69af7314d262495aa24784500c45247a is 175, key is test_row_0/A:col10/1731680242922/Put/seqid=0 2024-11-15T14:17:23,965 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 141d194711f441f788bea572ce6b868f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731680239573 2024-11-15T14:17:23,965 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting e36e5fd9d6204d69bfa13374ade3a6ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731680241776 2024-11-15T14:17:23,967 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f60b9a9513e14db3b4e4cb05d050912c_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f60b9a9513e14db3b4e4cb05d050912c_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:23,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/9eb2e20807804b25abc5588186120396, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:23,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/9eb2e20807804b25abc5588186120396 is 175, key is test_row_0/A:col10/1731680242962/Put/seqid=0 2024-11-15T14:17:23,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742478_1654 (size=32073) 2024-11-15T14:17:23,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742479_1655 (size=31255) 2024-11-15T14:17:23,977 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#C#compaction#553 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:23,977 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/9eb2e20807804b25abc5588186120396 2024-11-15T14:17:23,977 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/887d477035cc4d3b9abbe0457f60f88f is 50, key is test_row_0/C:col10/1731680242922/Put/seqid=0 2024-11-15T14:17:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/9fb647d959f246e6bbbde0d6d7142ca3 is 50, key is test_row_0/B:col10/1731680242962/Put/seqid=0 2024-11-15T14:17:23,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742480_1656 (size=13119) 2024-11-15T14:17:23,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742481_1657 (size=12301) 2024-11-15T14:17:23,992 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/9fb647d959f246e6bbbde0d6d7142ca3 2024-11-15T14:17:23,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/322addfd137b48d4b2238c118f93b166 is 50, key is test_row_0/C:col10/1731680242962/Put/seqid=0 2024-11-15T14:17:24,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742482_1658 (size=12301) 2024-11-15T14:17:24,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680304002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:24,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680304011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:24,021 INFO [Thread-2477 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2477. 2024-11-15T14:17:24,021 DEBUG [Thread-2477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:17:24,021 DEBUG [Thread-2477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,021 INFO [Thread-2471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2471. 2024-11-15T14:17:24,022 DEBUG [Thread-2477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,022 DEBUG [Thread-2471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:17:24,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:17:24,022 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,022 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:17:24,024 INFO [Thread-2473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2473. 2024-11-15T14:17:24,024 DEBUG [Thread-2473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:17:24,024 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,024 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,024 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:17:24,026 INFO [Thread-2469 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2469. 2024-11-15T14:17:24,026 DEBUG [Thread-2469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:17:24,026 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,026 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:17:24,026 INFO [Thread-2475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2475. 2024-11-15T14:17:24,026 DEBUG [Thread-2475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:17:24,026 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,026 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:17:24,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37634 deadline: 1731680304312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:24,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T14:17:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37680 deadline: 1731680304322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 2024-11-15T14:17:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 , the old value is region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T14:17:24,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., hostname=26754e923c4a,40177,1731680077530, seqNum=5 because the exception is null or not the one we care about 2024-11-15T14:17:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T14:17:24,382 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/69af7314d262495aa24784500c45247a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/69af7314d262495aa24784500c45247a 2024-11-15T14:17:24,385 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/A of e34e905c516b4c17b3db18acbd12c2a8 into 69af7314d262495aa24784500c45247a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:24,385 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:24,385 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/A, priority=13, startTime=1731680243542; duration=0sec 2024-11-15T14:17:24,385 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:24,385 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:A 2024-11-15T14:17:24,394 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/887d477035cc4d3b9abbe0457f60f88f as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/887d477035cc4d3b9abbe0457f60f88f 2024-11-15T14:17:24,398 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/C of e34e905c516b4c17b3db18acbd12c2a8 into 887d477035cc4d3b9abbe0457f60f88f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:24,399 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:24,399 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/C, priority=13, startTime=1731680243542; duration=0sec 2024-11-15T14:17:24,399 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:24,399 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:C 2024-11-15T14:17:24,401 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/322addfd137b48d4b2238c118f93b166 2024-11-15T14:17:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/9eb2e20807804b25abc5588186120396 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9eb2e20807804b25abc5588186120396 2024-11-15T14:17:24,406 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9eb2e20807804b25abc5588186120396, entries=150, sequenceid=330, filesize=30.5 K 2024-11-15T14:17:24,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/9fb647d959f246e6bbbde0d6d7142ca3 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/9fb647d959f246e6bbbde0d6d7142ca3 2024-11-15T14:17:24,410 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/9fb647d959f246e6bbbde0d6d7142ca3, entries=150, sequenceid=330, filesize=12.0 K 2024-11-15T14:17:24,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/322addfd137b48d4b2238c118f93b166 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/322addfd137b48d4b2238c118f93b166 2024-11-15T14:17:24,414 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/322addfd137b48d4b2238c118f93b166, entries=150, sequenceid=330, filesize=12.0 K 2024-11-15T14:17:24,415 INFO [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e34e905c516b4c17b3db18acbd12c2a8 in 864ms, sequenceid=330, compaction requested=false 2024-11-15T14:17:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26754e923c4a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-15T14:17:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-15T14:17:24,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-15T14:17:24,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1710 sec 2024-11-15T14:17:24,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.2960 sec 2024-11-15T14:17:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40177 {}] regionserver.HRegion(8855): Flush requested on e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:24,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-15T14:17:24,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:24,825 INFO [Thread-2464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2464. 2024-11-15T14:17:24,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:17:24,825 DEBUG [Thread-2464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:17:24,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:24,825 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:17:24,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:24,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T14:17:24,825 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:17:24,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fca2df08f2e14573a5d37ddb453c4b6f_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680243686/Put/seqid=0 2024-11-15T14:17:24,835 INFO [Thread-2466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2466. 2024-11-15T14:17:24,835 DEBUG [Thread-2466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:17:24,835 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,835 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:24,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:17:24,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742483_1659 (size=12454) 2024-11-15T14:17:25,238 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:25,242 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fca2df08f2e14573a5d37ddb453c4b6f_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fca2df08f2e14573a5d37ddb453c4b6f_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:25,242 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/9af6c5d13ccb4210b3d7273dc74a5b59, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:25,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/9af6c5d13ccb4210b3d7273dc74a5b59 is 175, key is test_row_0/A:col10/1731680243686/Put/seqid=0 2024-11-15T14:17:25,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742484_1660 (size=31255) 2024-11-15T14:17:25,246 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/9af6c5d13ccb4210b3d7273dc74a5b59 2024-11-15T14:17:25,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/15404e319671433eab750c06f1983b94 is 50, key is test_row_0/B:col10/1731680243686/Put/seqid=0 2024-11-15T14:17:25,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742485_1661 (size=12301) 2024-11-15T14:17:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T14:17:25,392 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:25,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/15404e319671433eab750c06f1983b94 2024-11-15T14:17:25,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/8607567035f44bfa972a6e9369182aac is 50, key is test_row_0/C:col10/1731680243686/Put/seqid=0 2024-11-15T14:17:25,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742486_1662 (size=12301) 2024-11-15T14:17:26,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/8607567035f44bfa972a6e9369182aac 2024-11-15T14:17:26,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/9af6c5d13ccb4210b3d7273dc74a5b59 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9af6c5d13ccb4210b3d7273dc74a5b59 2024-11-15T14:17:26,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9af6c5d13ccb4210b3d7273dc74a5b59, entries=150, sequenceid=369, filesize=30.5 K 2024-11-15T14:17:26,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/15404e319671433eab750c06f1983b94 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/15404e319671433eab750c06f1983b94 2024-11-15T14:17:26,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/15404e319671433eab750c06f1983b94, entries=150, sequenceid=369, filesize=12.0 K 2024-11-15T14:17:26,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/8607567035f44bfa972a6e9369182aac as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/8607567035f44bfa972a6e9369182aac 2024-11-15T14:17:26,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/8607567035f44bfa972a6e9369182aac, entries=150, sequenceid=369, filesize=12.0 K 2024-11-15T14:17:26,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=6.71 KB/6870 for e34e905c516b4c17b3db18acbd12c2a8 in 1271ms, sequenceid=369, compaction requested=true 2024-11-15T14:17:26,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:26,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T14:17:26,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:26,096 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:26,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T14:17:26,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:26,096 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:26,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34e905c516b4c17b3db18acbd12c2a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T14:17:26,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:26,097 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:26,097 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:26,097 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/A is initiating minor compaction (all files) 2024-11-15T14:17:26,097 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/B is initiating minor compaction (all files) 2024-11-15T14:17:26,097 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/A in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:26,097 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/B in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:26,097 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/69af7314d262495aa24784500c45247a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9eb2e20807804b25abc5588186120396, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9af6c5d13ccb4210b3d7273dc74a5b59] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=92.4 K 2024-11-15T14:17:26,097 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/0991f76acf964715be726236c2d362f0, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/9fb647d959f246e6bbbde0d6d7142ca3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/15404e319671433eab750c06f1983b94] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=36.8 K 2024-11-15T14:17:26,097 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:26,097 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. files: [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/69af7314d262495aa24784500c45247a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9eb2e20807804b25abc5588186120396, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9af6c5d13ccb4210b3d7273dc74a5b59] 2024-11-15T14:17:26,097 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0991f76acf964715be726236c2d362f0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731680241776 2024-11-15T14:17:26,097 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69af7314d262495aa24784500c45247a, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731680241776 2024-11-15T14:17:26,098 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fb647d959f246e6bbbde0d6d7142ca3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731680242956 2024-11-15T14:17:26,098 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eb2e20807804b25abc5588186120396, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731680242956 2024-11-15T14:17:26,098 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 15404e319671433eab750c06f1983b94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731680243680 2024-11-15T14:17:26,098 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9af6c5d13ccb4210b3d7273dc74a5b59, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731680243680 2024-11-15T14:17:26,106 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#B#compaction#559 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:26,107 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/4008bd6ed0544a2f82b177858c7af422 is 50, key is test_row_0/B:col10/1731680243686/Put/seqid=0 2024-11-15T14:17:26,109 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:26,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742487_1663 (size=13221) 2024-11-15T14:17:26,115 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111521cb966240834f3eab625eb8aa008ba6_e34e905c516b4c17b3db18acbd12c2a8 store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:26,139 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111521cb966240834f3eab625eb8aa008ba6_e34e905c516b4c17b3db18acbd12c2a8, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:26,139 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111521cb966240834f3eab625eb8aa008ba6_e34e905c516b4c17b3db18acbd12c2a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:26,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742488_1664 (size=4469) 2024-11-15T14:17:26,515 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/4008bd6ed0544a2f82b177858c7af422 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/4008bd6ed0544a2f82b177858c7af422 2024-11-15T14:17:26,520 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/B of e34e905c516b4c17b3db18acbd12c2a8 into 4008bd6ed0544a2f82b177858c7af422(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:26,520 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:26,520 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/B, priority=13, startTime=1731680246096; duration=0sec 2024-11-15T14:17:26,520 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T14:17:26,520 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:B 2024-11-15T14:17:26,520 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T14:17:26,521 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T14:17:26,521 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1541): e34e905c516b4c17b3db18acbd12c2a8/C is initiating minor compaction (all files) 2024-11-15T14:17:26,521 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34e905c516b4c17b3db18acbd12c2a8/C in TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:26,521 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/887d477035cc4d3b9abbe0457f60f88f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/322addfd137b48d4b2238c118f93b166, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/8607567035f44bfa972a6e9369182aac] into tmpdir=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp, totalSize=36.8 K 2024-11-15T14:17:26,522 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 887d477035cc4d3b9abbe0457f60f88f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731680241776 2024-11-15T14:17:26,522 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 322addfd137b48d4b2238c118f93b166, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731680242956 2024-11-15T14:17:26,522 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] compactions.Compactor(225): Compacting 8607567035f44bfa972a6e9369182aac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731680243680 2024-11-15T14:17:26,532 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#C#compaction#561 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:26,533 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/9173e17d56ad4f968f0d0f88e8a79381 is 50, key is test_row_0/C:col10/1731680243686/Put/seqid=0 2024-11-15T14:17:26,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742489_1665 (size=13221) 2024-11-15T14:17:26,557 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34e905c516b4c17b3db18acbd12c2a8#A#compaction#560 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-15T14:17:26,557 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/ef3f8dd9ef5249838aabba5ebc409447 is 175, key is test_row_0/A:col10/1731680243686/Put/seqid=0 2024-11-15T14:17:26,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742490_1666 (size=32175) 2024-11-15T14:17:26,941 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/9173e17d56ad4f968f0d0f88e8a79381 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/9173e17d56ad4f968f0d0f88e8a79381 2024-11-15T14:17:26,945 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/C of e34e905c516b4c17b3db18acbd12c2a8 into 9173e17d56ad4f968f0d0f88e8a79381(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:26,945 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:26,945 INFO [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/C, priority=13, startTime=1731680246096; duration=0sec 2024-11-15T14:17:26,945 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:26,945 DEBUG [RS:0;26754e923c4a:40177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:C 2024-11-15T14:17:26,969 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/ef3f8dd9ef5249838aabba5ebc409447 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/ef3f8dd9ef5249838aabba5ebc409447 2024-11-15T14:17:26,976 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34e905c516b4c17b3db18acbd12c2a8/A of e34e905c516b4c17b3db18acbd12c2a8 into ef3f8dd9ef5249838aabba5ebc409447(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T14:17:26,976 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34e905c516b4c17b3db18acbd12c2a8: 2024-11-15T14:17:26,976 INFO [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., storeName=e34e905c516b4c17b3db18acbd12c2a8/A, priority=13, startTime=1731680246095; duration=0sec 2024-11-15T14:17:26,976 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T14:17:26,976 DEBUG [RS:0;26754e923c4a:40177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34e905c516b4c17b3db18acbd12c2a8:A 2024-11-15T14:17:32,034 INFO [Thread-2460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2460. 2024-11-15T14:17:32,034 DEBUG [Thread-2460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:17:32,034 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:32,034 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:32,034 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:17:32,064 INFO [Thread-2462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2462. 2024-11-15T14:17:32,064 DEBUG [Thread-2462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:17:32,064 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:32,064 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:32,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:17:32,163 INFO [Thread-2458 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2458. 2024-11-15T14:17:32,163 DEBUG [Thread-2458 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T14:17:32,163 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:32,163 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:32,163 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:17:32,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T14:17:32,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-15T14:17:32,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-15T14:17:32,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-15T14:17:32,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-15T14:17:32,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-15T14:17:32,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T14:17:32,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2929 2024-11-15T14:17:32,163 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2878 2024-11-15T14:17:32,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2998 2024-11-15T14:17:32,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2938 2024-11-15T14:17:32,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2853 2024-11-15T14:17:32,164 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T14:17:32,164 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T14:17:32,164 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T14:17:32,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:32,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:32,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:17:32,165 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T14:17:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T14:17:32,167 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680252167"}]},"ts":"1731680252167"} 2024-11-15T14:17:32,168 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T14:17:32,169 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T14:17:32,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T14:17:32,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, UNASSIGN}] 2024-11-15T14:17:32,171 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, UNASSIGN 2024-11-15T14:17:32,172 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=e34e905c516b4c17b3db18acbd12c2a8, regionState=CLOSING, regionLocation=26754e923c4a,40177,1731680077530 2024-11-15T14:17:32,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, UNASSIGN because future has completed 2024-11-15T14:17:32,173 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T14:17:32,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530}] 2024-11-15T14:17:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T14:17:32,326 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:32,326 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T14:17:32,326 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing e34e905c516b4c17b3db18acbd12c2a8, disabling compactions & flushes 2024-11-15T14:17:32,326 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:32,326 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:32,326 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. after waiting 0 ms 2024-11-15T14:17:32,326 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:32,326 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing e34e905c516b4c17b3db18acbd12c2a8 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-15T14:17:32,326 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=A 2024-11-15T14:17:32,326 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:32,326 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=B 2024-11-15T14:17:32,326 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:32,326 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e34e905c516b4c17b3db18acbd12c2a8, store=C 2024-11-15T14:17:32,326 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T14:17:32,331 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111522f82cf0e1f34145b654d182b6f466f6_e34e905c516b4c17b3db18acbd12c2a8 is 50, key is test_row_0/A:col10/1731680244833/Put/seqid=0 2024-11-15T14:17:32,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742491_1667 (size=12454) 2024-11-15T14:17:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T14:17:32,735 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T14:17:32,740 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111522f82cf0e1f34145b654d182b6f466f6_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111522f82cf0e1f34145b654d182b6f466f6_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:32,740 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/71eac5577a0140c4a59755c7a7fdb82a, store: [table=TestAcidGuarantees family=A region=e34e905c516b4c17b3db18acbd12c2a8] 2024-11-15T14:17:32,741 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/71eac5577a0140c4a59755c7a7fdb82a is 175, key is test_row_0/A:col10/1731680244833/Put/seqid=0 2024-11-15T14:17:32,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742492_1668 (size=31255) 2024-11-15T14:17:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T14:17:33,146 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/71eac5577a0140c4a59755c7a7fdb82a 2024-11-15T14:17:33,155 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/e0d87aebd47c49f3ac7e31faa6c37910 is 50, key is test_row_0/B:col10/1731680244833/Put/seqid=0 2024-11-15T14:17:33,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742493_1669 (size=12301) 2024-11-15T14:17:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T14:17:33,559 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/e0d87aebd47c49f3ac7e31faa6c37910 2024-11-15T14:17:33,564 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/858e20c9cd384c19955a69ee720ca378 is 50, key is test_row_0/C:col10/1731680244833/Put/seqid=0 2024-11-15T14:17:33,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742494_1670 (size=12301) 2024-11-15T14:17:33,968 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/858e20c9cd384c19955a69ee720ca378 2024-11-15T14:17:33,971 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/A/71eac5577a0140c4a59755c7a7fdb82a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/71eac5577a0140c4a59755c7a7fdb82a 2024-11-15T14:17:33,974 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/71eac5577a0140c4a59755c7a7fdb82a, entries=150, sequenceid=379, filesize=30.5 K 2024-11-15T14:17:33,974 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/B/e0d87aebd47c49f3ac7e31faa6c37910 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/e0d87aebd47c49f3ac7e31faa6c37910 2024-11-15T14:17:33,977 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/e0d87aebd47c49f3ac7e31faa6c37910, entries=150, sequenceid=379, filesize=12.0 K 2024-11-15T14:17:33,977 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/.tmp/C/858e20c9cd384c19955a69ee720ca378 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/858e20c9cd384c19955a69ee720ca378 2024-11-15T14:17:33,979 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/858e20c9cd384c19955a69ee720ca378, entries=150, sequenceid=379, filesize=12.0 K 2024-11-15T14:17:33,980 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e34e905c516b4c17b3db18acbd12c2a8 in 1654ms, sequenceid=379, compaction requested=false 2024-11-15T14:17:33,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/d5a81745ab254d9d91eb3c33ab08f9d3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/1733618375f54c6fbdc6542fd4e8e242, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/913d1123963d44af9bef5ea03a8e5178, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/6484132b33da456db498ee9750c426e0, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/94abea38d5a74b3ab9d0a98dd8d7b07d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/c3bb533cbb2146139c69817fdd02b3d2, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/835d3f0eaef4465c9ab4ec3652e6b9be, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/697b14788bb44155bc29b29b1bdeae3b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/2e38cd54965f4d4697bfc77a9357e290, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/57dd081571ac4d09888202c9140fb6c0, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/2610e7db0c99497680122e0f2988f063, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/019182046a6744ef9ada0becf88e46af, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/d3302a6c859a43f6abc8d607b61436b4, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f44f51829f0543b69fb0c9141e4d6cee, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e22201c04c494cccae7dff33f24285ea, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/ac37307d2f334c999f592f3ae9334b68, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e0fe0aedcee34489b7cb0f5e4f6c8762, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/8b262803a6e64defb24c5cd86c98815d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/c24596ff3577465cb825d2e49777b621, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f304b802c4034f12b127f03148270789, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/b4b0bd76f2284738b6c3f6135154f88d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/dd9b8d17a902484f9c806d17104ff63d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e084c48558db47c18aaead21696782d5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/638583b87f5743bbadf05ece9c006060, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/69af7314d262495aa24784500c45247a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9eb2e20807804b25abc5588186120396, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9af6c5d13ccb4210b3d7273dc74a5b59] to archive 2024-11-15T14:17:33,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:17:33,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/d5a81745ab254d9d91eb3c33ab08f9d3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/d5a81745ab254d9d91eb3c33ab08f9d3 2024-11-15T14:17:33,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/1733618375f54c6fbdc6542fd4e8e242 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/1733618375f54c6fbdc6542fd4e8e242 2024-11-15T14:17:33,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/913d1123963d44af9bef5ea03a8e5178 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/913d1123963d44af9bef5ea03a8e5178 2024-11-15T14:17:33,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/6484132b33da456db498ee9750c426e0 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/6484132b33da456db498ee9750c426e0 2024-11-15T14:17:33,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/94abea38d5a74b3ab9d0a98dd8d7b07d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/94abea38d5a74b3ab9d0a98dd8d7b07d 2024-11-15T14:17:33,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/c3bb533cbb2146139c69817fdd02b3d2 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/c3bb533cbb2146139c69817fdd02b3d2 2024-11-15T14:17:33,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/835d3f0eaef4465c9ab4ec3652e6b9be to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/835d3f0eaef4465c9ab4ec3652e6b9be 2024-11-15T14:17:33,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/697b14788bb44155bc29b29b1bdeae3b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/697b14788bb44155bc29b29b1bdeae3b 2024-11-15T14:17:33,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/2e38cd54965f4d4697bfc77a9357e290 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/2e38cd54965f4d4697bfc77a9357e290 2024-11-15T14:17:33,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/57dd081571ac4d09888202c9140fb6c0 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/57dd081571ac4d09888202c9140fb6c0 2024-11-15T14:17:33,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/2610e7db0c99497680122e0f2988f063 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/2610e7db0c99497680122e0f2988f063 2024-11-15T14:17:33,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/019182046a6744ef9ada0becf88e46af to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/019182046a6744ef9ada0becf88e46af 2024-11-15T14:17:33,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/d3302a6c859a43f6abc8d607b61436b4 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/d3302a6c859a43f6abc8d607b61436b4 2024-11-15T14:17:33,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f44f51829f0543b69fb0c9141e4d6cee to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f44f51829f0543b69fb0c9141e4d6cee 2024-11-15T14:17:33,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e22201c04c494cccae7dff33f24285ea to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e22201c04c494cccae7dff33f24285ea 2024-11-15T14:17:33,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/ac37307d2f334c999f592f3ae9334b68 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/ac37307d2f334c999f592f3ae9334b68 2024-11-15T14:17:33,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e0fe0aedcee34489b7cb0f5e4f6c8762 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e0fe0aedcee34489b7cb0f5e4f6c8762 2024-11-15T14:17:33,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/8b262803a6e64defb24c5cd86c98815d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/8b262803a6e64defb24c5cd86c98815d 2024-11-15T14:17:33,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/c24596ff3577465cb825d2e49777b621 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/c24596ff3577465cb825d2e49777b621 2024-11-15T14:17:33,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f304b802c4034f12b127f03148270789 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/f304b802c4034f12b127f03148270789 2024-11-15T14:17:33,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/b4b0bd76f2284738b6c3f6135154f88d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/b4b0bd76f2284738b6c3f6135154f88d 2024-11-15T14:17:33,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/dd9b8d17a902484f9c806d17104ff63d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/dd9b8d17a902484f9c806d17104ff63d 2024-11-15T14:17:33,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e084c48558db47c18aaead21696782d5 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/e084c48558db47c18aaead21696782d5 2024-11-15T14:17:33,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/638583b87f5743bbadf05ece9c006060 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/638583b87f5743bbadf05ece9c006060 2024-11-15T14:17:34,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/69af7314d262495aa24784500c45247a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/69af7314d262495aa24784500c45247a 2024-11-15T14:17:34,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9eb2e20807804b25abc5588186120396 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9eb2e20807804b25abc5588186120396 2024-11-15T14:17:34,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9af6c5d13ccb4210b3d7273dc74a5b59 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/9af6c5d13ccb4210b3d7273dc74a5b59 2024-11-15T14:17:34,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/841949ef840f40c69164e10e389d49ec, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/4f3cd6fbf1704423a1d040f524a7fc30, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/03e15459b2764977903f1066d6df6dde, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/163e6f766e8d4dcf87007c6341256d02, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/fb1a32034b3e491bb536d654a33c9494, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/0ce77271703a47fc9d5a5b635932dae3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1844b1375e1f4556878c246895e48255, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/f1614da8b57146be9492a5160e224808, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/88cbda2006c94b239102fa2fcd88dec6, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/5d9109253dc2416c841f9a47376392a1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1ab83572cd23456cbcdc160c48e97083, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/5bfa108129ec4cb2a8a73efbef20da41, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/105a74843ed0487bbbc1fa1721ad87ab, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/a9a5e8bae88c484b99b9205e31494d54, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/c48d90f88b6b435ba3eed7e48ecc03db, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/cb3aa5c0817c4264862480110ffae695, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1ae2163ba0364d1694ab89368b71f9a7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/966c0ef865934f1882c860622c9379d1, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/75413411c5694dfcb4ae198f71eb89fb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/02eb1b26691b4958849946eb6a0d1694, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/cdb618a2bbf945e1bbeabc6578be4102, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/29c8529d72794ad6bf873c877c1c2f5e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/c1e2233044f442328a9a4c436d44c2ca, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/0991f76acf964715be726236c2d362f0, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/315336fb372946bd8189955aa171ffa3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/9fb647d959f246e6bbbde0d6d7142ca3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/15404e319671433eab750c06f1983b94] to archive 2024-11-15T14:17:34,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:17:34,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/841949ef840f40c69164e10e389d49ec to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/841949ef840f40c69164e10e389d49ec 2024-11-15T14:17:34,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/4f3cd6fbf1704423a1d040f524a7fc30 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/4f3cd6fbf1704423a1d040f524a7fc30 2024-11-15T14:17:34,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/03e15459b2764977903f1066d6df6dde to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/03e15459b2764977903f1066d6df6dde 2024-11-15T14:17:34,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/163e6f766e8d4dcf87007c6341256d02 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/163e6f766e8d4dcf87007c6341256d02 2024-11-15T14:17:34,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/fb1a32034b3e491bb536d654a33c9494 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/fb1a32034b3e491bb536d654a33c9494 2024-11-15T14:17:34,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/0ce77271703a47fc9d5a5b635932dae3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/0ce77271703a47fc9d5a5b635932dae3 2024-11-15T14:17:34,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1844b1375e1f4556878c246895e48255 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1844b1375e1f4556878c246895e48255 2024-11-15T14:17:34,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/f1614da8b57146be9492a5160e224808 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/f1614da8b57146be9492a5160e224808 2024-11-15T14:17:34,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/88cbda2006c94b239102fa2fcd88dec6 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/88cbda2006c94b239102fa2fcd88dec6 2024-11-15T14:17:34,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/5d9109253dc2416c841f9a47376392a1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/5d9109253dc2416c841f9a47376392a1 2024-11-15T14:17:34,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1ab83572cd23456cbcdc160c48e97083 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1ab83572cd23456cbcdc160c48e97083 2024-11-15T14:17:34,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/5bfa108129ec4cb2a8a73efbef20da41 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/5bfa108129ec4cb2a8a73efbef20da41 2024-11-15T14:17:34,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/105a74843ed0487bbbc1fa1721ad87ab to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/105a74843ed0487bbbc1fa1721ad87ab 2024-11-15T14:17:34,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/a9a5e8bae88c484b99b9205e31494d54 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/a9a5e8bae88c484b99b9205e31494d54 2024-11-15T14:17:34,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/c48d90f88b6b435ba3eed7e48ecc03db to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/c48d90f88b6b435ba3eed7e48ecc03db 2024-11-15T14:17:34,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/cb3aa5c0817c4264862480110ffae695 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/cb3aa5c0817c4264862480110ffae695 2024-11-15T14:17:34,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1ae2163ba0364d1694ab89368b71f9a7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/1ae2163ba0364d1694ab89368b71f9a7 2024-11-15T14:17:34,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/966c0ef865934f1882c860622c9379d1 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/966c0ef865934f1882c860622c9379d1 2024-11-15T14:17:34,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/75413411c5694dfcb4ae198f71eb89fb to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/75413411c5694dfcb4ae198f71eb89fb 2024-11-15T14:17:34,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/02eb1b26691b4958849946eb6a0d1694 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/02eb1b26691b4958849946eb6a0d1694 2024-11-15T14:17:34,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/cdb618a2bbf945e1bbeabc6578be4102 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/cdb618a2bbf945e1bbeabc6578be4102 2024-11-15T14:17:34,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/29c8529d72794ad6bf873c877c1c2f5e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/29c8529d72794ad6bf873c877c1c2f5e 2024-11-15T14:17:34,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/c1e2233044f442328a9a4c436d44c2ca to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/c1e2233044f442328a9a4c436d44c2ca 2024-11-15T14:17:34,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/0991f76acf964715be726236c2d362f0 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/0991f76acf964715be726236c2d362f0 2024-11-15T14:17:34,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/315336fb372946bd8189955aa171ffa3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/315336fb372946bd8189955aa171ffa3 2024-11-15T14:17:34,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/9fb647d959f246e6bbbde0d6d7142ca3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/9fb647d959f246e6bbbde0d6d7142ca3 2024-11-15T14:17:34,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/15404e319671433eab750c06f1983b94 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/15404e319671433eab750c06f1983b94 2024-11-15T14:17:34,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e35d32edb51d4f4c9460ac52ed2d88cb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/4b5fa06742034bfeb0556d2f1a45216b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/4523ff94c3654eb294fb55cb4cb5ce8b, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/a2cc4f8cba6847bd9f3077bfd996f706, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/37ac3cf93bf14c8886f5c87066cd7038, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/6489190063594281ba5c4490538b30a0, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/fae1137e994740b7b02954c4b5dcdeeb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/fd4f80e689a04ecb8f1cb86381862496, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/9019c331ab634d73a04ce0757da835ca, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/c916c4f153714df5977d88642267ccd5, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/1c8b918c870340278bb19e43c7dd546e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/acf8bc02ec0f40bdafb6d928091901c7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/387a0de2b19a4d44bcb895b761113a6a, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/783455c68a644fda861c0ac05b72a8eb, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/dd101b606d35435e843fd8e46a143bc3, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e9a678385079423591f862fa2b4ade4c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/60e06f11c6e9414ea86dc3a4c147b95c, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/0333c0661d8d4410a8514cedcdd7cdc8, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/cbcd464fb72a4e328dc4ce918db0de9e, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/edd7ae9680b04ff695de0fc6012114a7, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/420ab49b4b8d47838f3ac11c3f6a0fff, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/b3f8f061a609432e946a6c647543209d, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/141d194711f441f788bea572ce6b868f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/887d477035cc4d3b9abbe0457f60f88f, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e36e5fd9d6204d69bfa13374ade3a6ba, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/322addfd137b48d4b2238c118f93b166, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/8607567035f44bfa972a6e9369182aac] to archive 2024-11-15T14:17:34,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T14:17:34,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e35d32edb51d4f4c9460ac52ed2d88cb to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e35d32edb51d4f4c9460ac52ed2d88cb 2024-11-15T14:17:34,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/4b5fa06742034bfeb0556d2f1a45216b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/4b5fa06742034bfeb0556d2f1a45216b 2024-11-15T14:17:34,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/4523ff94c3654eb294fb55cb4cb5ce8b to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/4523ff94c3654eb294fb55cb4cb5ce8b 2024-11-15T14:17:34,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/a2cc4f8cba6847bd9f3077bfd996f706 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/a2cc4f8cba6847bd9f3077bfd996f706 2024-11-15T14:17:34,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/37ac3cf93bf14c8886f5c87066cd7038 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/37ac3cf93bf14c8886f5c87066cd7038 2024-11-15T14:17:34,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/6489190063594281ba5c4490538b30a0 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/6489190063594281ba5c4490538b30a0 2024-11-15T14:17:34,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/fae1137e994740b7b02954c4b5dcdeeb to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/fae1137e994740b7b02954c4b5dcdeeb 2024-11-15T14:17:34,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/fd4f80e689a04ecb8f1cb86381862496 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/fd4f80e689a04ecb8f1cb86381862496 2024-11-15T14:17:34,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/9019c331ab634d73a04ce0757da835ca to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/9019c331ab634d73a04ce0757da835ca 2024-11-15T14:17:34,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/c916c4f153714df5977d88642267ccd5 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/c916c4f153714df5977d88642267ccd5 2024-11-15T14:17:34,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/1c8b918c870340278bb19e43c7dd546e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/1c8b918c870340278bb19e43c7dd546e 2024-11-15T14:17:34,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/acf8bc02ec0f40bdafb6d928091901c7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/acf8bc02ec0f40bdafb6d928091901c7 2024-11-15T14:17:34,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/387a0de2b19a4d44bcb895b761113a6a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/387a0de2b19a4d44bcb895b761113a6a 2024-11-15T14:17:34,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/783455c68a644fda861c0ac05b72a8eb to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/783455c68a644fda861c0ac05b72a8eb 2024-11-15T14:17:34,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/dd101b606d35435e843fd8e46a143bc3 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/dd101b606d35435e843fd8e46a143bc3 2024-11-15T14:17:34,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e9a678385079423591f862fa2b4ade4c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e9a678385079423591f862fa2b4ade4c 2024-11-15T14:17:34,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/60e06f11c6e9414ea86dc3a4c147b95c to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/60e06f11c6e9414ea86dc3a4c147b95c 2024-11-15T14:17:34,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/0333c0661d8d4410a8514cedcdd7cdc8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/0333c0661d8d4410a8514cedcdd7cdc8 2024-11-15T14:17:34,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/cbcd464fb72a4e328dc4ce918db0de9e to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/cbcd464fb72a4e328dc4ce918db0de9e 2024-11-15T14:17:34,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/edd7ae9680b04ff695de0fc6012114a7 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/edd7ae9680b04ff695de0fc6012114a7 2024-11-15T14:17:34,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/420ab49b4b8d47838f3ac11c3f6a0fff to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/420ab49b4b8d47838f3ac11c3f6a0fff 2024-11-15T14:17:34,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/b3f8f061a609432e946a6c647543209d to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/b3f8f061a609432e946a6c647543209d 2024-11-15T14:17:34,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/141d194711f441f788bea572ce6b868f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/141d194711f441f788bea572ce6b868f 2024-11-15T14:17:34,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/887d477035cc4d3b9abbe0457f60f88f to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/887d477035cc4d3b9abbe0457f60f88f 2024-11-15T14:17:34,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e36e5fd9d6204d69bfa13374ade3a6ba to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/e36e5fd9d6204d69bfa13374ade3a6ba 2024-11-15T14:17:34,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/322addfd137b48d4b2238c118f93b166 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/322addfd137b48d4b2238c118f93b166 2024-11-15T14:17:34,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/8607567035f44bfa972a6e9369182aac to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/8607567035f44bfa972a6e9369182aac 2024-11-15T14:17:34,068 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/recovered.edits/382.seqid, newMaxSeqId=382, maxSeqId=4 2024-11-15T14:17:34,069 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. 2024-11-15T14:17:34,069 DEBUG [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for e34e905c516b4c17b3db18acbd12c2a8: Waiting for close lock at 1731680252326Running coprocessor pre-close hooks at 1731680252326Disabling compacts and flushes for region at 1731680252326Disabling writes for close at 1731680252326Obtaining lock to block concurrent updates at 1731680252326Preparing flush snapshotting stores in e34e905c516b4c17b3db18acbd12c2a8 at 1731680252326Finished memstore snapshotting TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731680252326Flushing stores of TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8. at 1731680252327 (+1 ms)Flushing e34e905c516b4c17b3db18acbd12c2a8/A: creating writer at 1731680252327Flushing mob file e34e905c516b4c17b3db18acbd12c2a8/A: appending metadata at 1731680252331 (+4 ms)Flushing mob file e34e905c516b4c17b3db18acbd12c2a8/A: closing flushed file at 1731680252331Flushing e34e905c516b4c17b3db18acbd12c2a8/A: appending metadata at 1731680252741 (+410 ms)Flushing e34e905c516b4c17b3db18acbd12c2a8/A: closing flushed file at 1731680252741Flushing e34e905c516b4c17b3db18acbd12c2a8/B: creating writer at 1731680253151 (+410 ms)Flushing e34e905c516b4c17b3db18acbd12c2a8/B: appending metadata at 1731680253154 (+3 ms)Flushing e34e905c516b4c17b3db18acbd12c2a8/B: closing flushed file at 1731680253154Flushing e34e905c516b4c17b3db18acbd12c2a8/C: creating writer at 1731680253562 (+408 ms)Flushing e34e905c516b4c17b3db18acbd12c2a8/C: appending metadata at 1731680253564 (+2 ms)Flushing e34e905c516b4c17b3db18acbd12c2a8/C: closing flushed file at 1731680253564Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b411630: reopening flushed file at 1731680253971 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@639cd461: reopening flushed file at 1731680253974 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21a6fca3: reopening flushed file at 1731680253977 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e34e905c516b4c17b3db18acbd12c2a8 in 1654ms, sequenceid=379, compaction requested=false at 1731680253980 (+3 ms)Writing region close event to WAL at 1731680254065 (+85 ms)Running coprocessor post-close hooks at 1731680254069 (+4 ms)Closed at 1731680254069 2024-11-15T14:17:34,072 INFO [RS_CLOSE_REGION-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,073 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=e34e905c516b4c17b3db18acbd12c2a8, regionState=CLOSED 2024-11-15T14:17:34,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 because future has completed 2024-11-15T14:17:34,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-15T14:17:34,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure e34e905c516b4c17b3db18acbd12c2a8, server=26754e923c4a,40177,1731680077530 in 1.9030 sec 2024-11-15T14:17:34,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-15T14:17:34,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34e905c516b4c17b3db18acbd12c2a8, UNASSIGN in 1.9080 sec 2024-11-15T14:17:34,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-15T14:17:34,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9120 sec 2024-11-15T14:17:34,084 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731680254084"}]},"ts":"1731680254084"} 2024-11-15T14:17:34,085 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T14:17:34,085 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T14:17:34,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9210 sec 2024-11-15T14:17:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T14:17:34,323 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:34,324 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T14:17:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:34,327 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:34,328 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:34,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T14:17:34,334 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,336 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C, FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/recovered.edits] 2024-11-15T14:17:34,338 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/71eac5577a0140c4a59755c7a7fdb82a to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/71eac5577a0140c4a59755c7a7fdb82a 2024-11-15T14:17:34,339 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/ef3f8dd9ef5249838aabba5ebc409447 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/A/ef3f8dd9ef5249838aabba5ebc409447 2024-11-15T14:17:34,341 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/4008bd6ed0544a2f82b177858c7af422 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/4008bd6ed0544a2f82b177858c7af422 2024-11-15T14:17:34,341 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/e0d87aebd47c49f3ac7e31faa6c37910 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/B/e0d87aebd47c49f3ac7e31faa6c37910 2024-11-15T14:17:34,343 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/858e20c9cd384c19955a69ee720ca378 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/858e20c9cd384c19955a69ee720ca378 2024-11-15T14:17:34,344 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/9173e17d56ad4f968f0d0f88e8a79381 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/C/9173e17d56ad4f968f0d0f88e8a79381 2024-11-15T14:17:34,345 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/recovered.edits/382.seqid to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8/recovered.edits/382.seqid 2024-11-15T14:17:34,346 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/default/TestAcidGuarantees/e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,346 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T14:17:34,346 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T14:17:34,347 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T14:17:34,349 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111504abeeff70ac46d88bffb994b8b27294_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111504abeeff70ac46d88bffb994b8b27294_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,350 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111522f82cf0e1f34145b654d182b6f466f6_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111522f82cf0e1f34145b654d182b6f466f6_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,351 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111526925c4196064505b1aa347dd31cdafe_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111526925c4196064505b1aa347dd31cdafe_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,351 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152a94246fc5b24a0fad66ee58ef93dcc3_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152a94246fc5b24a0fad66ee58ef93dcc3_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,352 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153ad9a003a6db4572ae4457f6cc358054_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153ad9a003a6db4572ae4457f6cc358054_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,353 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153c443cbdc93b481c881e5b07431f2657_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153c443cbdc93b481c881e5b07431f2657_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,353 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f92dca82ee842b1ad3300cbcd8c698f_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f92dca82ee842b1ad3300cbcd8c698f_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,354 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115712b9a546d72409498d465f966526caa_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115712b9a546d72409498d465f966526caa_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,355 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115892d0a0bd1424ec190f2ed63483fb12d_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115892d0a0bd1424ec190f2ed63483fb12d_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,356 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159b618aeddbc8475080a7fca693f02451_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159b618aeddbc8475080a7fca693f02451_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,356 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115af003f1ace9047f99b96e531565b6cca_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115af003f1ace9047f99b96e531565b6cca_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,357 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b694a30fcff04d7a8c2d4b64ab45287c_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b694a30fcff04d7a8c2d4b64ab45287c_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,358 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bd9e5ec7f3904b40a31bee0f4e3d13bb_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bd9e5ec7f3904b40a31bee0f4e3d13bb_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,358 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf37d0e22370404fb1fa48e36b8b2c80_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf37d0e22370404fb1fa48e36b8b2c80_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,359 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d1d8126da8de47aa99ecebfe074a0a42_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d1d8126da8de47aa99ecebfe074a0a42_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,360 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e0ab3eeab1f346f88f56d5df63dcdd09_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e0ab3eeab1f346f88f56d5df63dcdd09_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,361 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e660fc6e42d143d49a3a5794df49689e_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e660fc6e42d143d49a3a5794df49689e_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,361 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f3dc681f0e834fc68b30e8a2665ee309_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f3dc681f0e834fc68b30e8a2665ee309_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,362 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f60b9a9513e14db3b4e4cb05d050912c_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f60b9a9513e14db3b4e4cb05d050912c_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,363 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fca2df08f2e14573a5d37ddb453c4b6f_e34e905c516b4c17b3db18acbd12c2a8 to hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fca2df08f2e14573a5d37ddb453c4b6f_e34e905c516b4c17b3db18acbd12c2a8 2024-11-15T14:17:34,363 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T14:17:34,365 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:34,367 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T14:17:34,370 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T14:17:34,371 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:34,371 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T14:17:34,371 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731680254371"}]},"ts":"9223372036854775807"} 2024-11-15T14:17:34,373 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T14:17:34,373 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e34e905c516b4c17b3db18acbd12c2a8, NAME => 'TestAcidGuarantees,,1731680220894.e34e905c516b4c17b3db18acbd12c2a8.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T14:17:34,373 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T14:17:34,373 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731680254373"}]},"ts":"9223372036854775807"} 2024-11-15T14:17:34,374 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T14:17:34,375 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T14:17:34,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-15T14:17:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T14:17:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T14:17:34,442 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T14:17:34,451 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 249), OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=702 (was 814), ProcessCount=11 (was 11), AvailableMemoryMB=630 (was 533) - AvailableMemoryMB LEAK? - 2024-11-15T14:17:34,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T14:17:34,451 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T14:17:34,451 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T14:17:34,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:34,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:34,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T14:17:34,452 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T14:17:34,452 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1336910459, stopped=false 2024-11-15T14:17:34,452 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=26754e923c4a,35807,1731680075796 2024-11-15T14:17:34,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T14:17:34,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T14:17:34,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:17:34,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:17:34,487 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T14:17:34,487 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T14:17:34,488 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T14:17:34,488 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T14:17:34,488 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T14:17:34,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:34,488 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '26754e923c4a,40177,1731680077530' ***** 2024-11-15T14:17:34,488 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T14:17:34,488 INFO [RS:0;26754e923c4a:40177 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T14:17:34,489 INFO [RS:0;26754e923c4a:40177 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T14:17:34,489 INFO [RS:0;26754e923c4a:40177 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T14:17:34,489 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T14:17:34,489 INFO [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(959): stopping server 26754e923c4a,40177,1731680077530 2024-11-15T14:17:34,489 INFO [RS:0;26754e923c4a:40177 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T14:17:34,489 INFO [RS:0;26754e923c4a:40177 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;26754e923c4a:40177. 2024-11-15T14:17:34,489 DEBUG [RS:0;26754e923c4a:40177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T14:17:34,489 DEBUG [RS:0;26754e923c4a:40177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:34,489 INFO [RS:0;26754e923c4a:40177 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T14:17:34,489 INFO [RS:0;26754e923c4a:40177 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T14:17:34,489 INFO [RS:0;26754e923c4a:40177 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T14:17:34,489 INFO [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T14:17:34,490 INFO [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T14:17:34,490 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T14:17:34,490 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T14:17:34,490 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T14:17:34,490 INFO [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T14:17:34,490 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T14:17:34,490 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T14:17:34,490 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T14:17:34,490 INFO [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-15T14:17:34,494 INFO [regionserver/26754e923c4a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T14:17:34,495 INFO [regionserver/26754e923c4a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T14:17:34,495 INFO [regionserver/26754e923c4a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T14:17:34,515 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/.tmp/info/263a1fc1369e4e9ab20828722075137c is 95, key is TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68./info:/1731680106698/DeleteFamily/seqid=0 2024-11-15T14:17:34,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742495_1671 (size=6514) 2024-11-15T14:17:34,691 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T14:17:34,812 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T14:17:34,891 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T14:17:34,920 INFO [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/.tmp/info/263a1fc1369e4e9ab20828722075137c 2024-11-15T14:17:34,947 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/.tmp/ns/f491889c0866443d97bfaab348fa737d is 93, key is TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68./ns:/1731680106662/DeleteFamily/seqid=0 2024-11-15T14:17:34,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742496_1672 (size=5906) 2024-11-15T14:17:35,091 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T14:17:35,291 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T14:17:35,351 INFO [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/.tmp/ns/f491889c0866443d97bfaab348fa737d 2024-11-15T14:17:35,371 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/.tmp/rep_barrier/f85f4b001829418ab2dfccfdc8048f7a is 102, key is TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68./rep_barrier:/1731680106662/DeleteFamily/seqid=0 2024-11-15T14:17:35,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742497_1673 (size=6025) 2024-11-15T14:17:35,492 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T14:17:35,692 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T14:17:35,775 INFO [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/.tmp/rep_barrier/f85f4b001829418ab2dfccfdc8048f7a 2024-11-15T14:17:35,803 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/.tmp/table/4c370d07a63d4da09d6c2759d0c42d31 is 96, key is TestAcidGuarantees,,1731680081956.785aa89faccd90f1bafb73964185be68./table:/1731680106662/DeleteFamily/seqid=0 2024-11-15T14:17:35,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742498_1674 (size=5936) 2024-11-15T14:17:35,892 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T14:17:36,092 DEBUG [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T14:17:36,207 INFO [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/.tmp/table/4c370d07a63d4da09d6c2759d0c42d31 2024-11-15T14:17:36,212 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/.tmp/info/263a1fc1369e4e9ab20828722075137c as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/info/263a1fc1369e4e9ab20828722075137c 2024-11-15T14:17:36,216 INFO [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/info/263a1fc1369e4e9ab20828722075137c, entries=12, sequenceid=90, filesize=6.4 K 2024-11-15T14:17:36,217 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/.tmp/ns/f491889c0866443d97bfaab348fa737d as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/ns/f491889c0866443d97bfaab348fa737d 2024-11-15T14:17:36,221 INFO [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/ns/f491889c0866443d97bfaab348fa737d, entries=8, sequenceid=90, filesize=5.8 K 2024-11-15T14:17:36,222 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/.tmp/rep_barrier/f85f4b001829418ab2dfccfdc8048f7a as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/rep_barrier/f85f4b001829418ab2dfccfdc8048f7a 2024-11-15T14:17:36,225 INFO [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/rep_barrier/f85f4b001829418ab2dfccfdc8048f7a, entries=6, sequenceid=90, filesize=5.9 K 2024-11-15T14:17:36,226 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/.tmp/table/4c370d07a63d4da09d6c2759d0c42d31 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/table/4c370d07a63d4da09d6c2759d0c42d31 2024-11-15T14:17:36,230 INFO [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/table/4c370d07a63d4da09d6c2759d0c42d31, entries=7, sequenceid=90, filesize=5.8 K 2024-11-15T14:17:36,231 INFO [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1741ms, sequenceid=90, compaction requested=false 2024-11-15T14:17:36,236 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-15T14:17:36,237 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T14:17:36,237 INFO [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T14:17:36,237 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731680254490Running coprocessor pre-close hooks at 1731680254490Disabling compacts and flushes for region at 1731680254490Disabling writes for close at 1731680254490Obtaining lock to block concurrent updates at 1731680254491 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731680254491Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731680254491Flushing stores of hbase:meta,,1.1588230740 at 1731680254492 (+1 ms)Flushing 1588230740/info: creating writer at 1731680254492Flushing 1588230740/info: appending metadata at 1731680254515 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731680254515Flushing 1588230740/ns: creating writer at 1731680254928 (+413 ms)Flushing 1588230740/ns: appending metadata at 1731680254947 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731680254947Flushing 1588230740/rep_barrier: creating writer at 1731680255355 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731680255370 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731680255370Flushing 1588230740/table: creating writer at 1731680255783 (+413 ms)Flushing 1588230740/table: appending metadata at 1731680255803 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731680255803Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9e43241: reopening flushed file at 1731680256211 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2240028: reopening flushed file at 1731680256217 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d6fcaa5: reopening flushed file at 1731680256221 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@723145e: reopening flushed file at 1731680256225 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1741ms, sequenceid=90, compaction requested=false at 1731680256231 (+6 ms)Writing region close event to WAL at 1731680256233 (+2 ms)Running coprocessor post-close hooks at 1731680256237 (+4 ms)Closed at 1731680256237 2024-11-15T14:17:36,237 DEBUG [RS_CLOSE_META-regionserver/26754e923c4a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T14:17:36,293 INFO [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(976): stopping server 26754e923c4a,40177,1731680077530; all regions closed. 2024-11-15T14:17:36,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741834_1010 (size=25344) 2024-11-15T14:17:36,305 DEBUG [RS:0;26754e923c4a:40177 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/oldWALs 2024-11-15T14:17:36,305 INFO [RS:0;26754e923c4a:40177 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26754e923c4a%2C40177%2C1731680077530.meta:.meta(num 1731680081257) 2024-11-15T14:17:36,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741833_1009 (size=16344337) 2024-11-15T14:17:36,311 DEBUG [RS:0;26754e923c4a:40177 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/oldWALs 2024-11-15T14:17:36,311 INFO [RS:0;26754e923c4a:40177 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26754e923c4a%2C40177%2C1731680077530:(num 1731680080683) 2024-11-15T14:17:36,311 DEBUG [RS:0;26754e923c4a:40177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T14:17:36,311 INFO [RS:0;26754e923c4a:40177 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T14:17:36,311 INFO [RS:0;26754e923c4a:40177 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T14:17:36,311 INFO [RS:0;26754e923c4a:40177 {}] hbase.ChoreService(370): Chore service for: regionserver/26754e923c4a:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T14:17:36,312 INFO [RS:0;26754e923c4a:40177 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T14:17:36,312 INFO [regionserver/26754e923c4a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T14:17:36,313 INFO [RS:0;26754e923c4a:40177 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40177 2024-11-15T14:17:36,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/26754e923c4a,40177,1731680077530 2024-11-15T14:17:36,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T14:17:36,323 INFO [RS:0;26754e923c4a:40177 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T14:17:36,331 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [26754e923c4a,40177,1731680077530] 2024-11-15T14:17:36,340 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/26754e923c4a,40177,1731680077530 already deleted, retry=false 2024-11-15T14:17:36,340 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 26754e923c4a,40177,1731680077530 expired; onlineServers=0 2024-11-15T14:17:36,340 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '26754e923c4a,35807,1731680075796' ***** 2024-11-15T14:17:36,340 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T14:17:36,341 INFO [M:0;26754e923c4a:35807 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T14:17:36,341 INFO [M:0;26754e923c4a:35807 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T14:17:36,341 DEBUG [M:0;26754e923c4a:35807 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T14:17:36,341 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T14:17:36,341 DEBUG [M:0;26754e923c4a:35807 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T14:17:36,341 DEBUG [master/26754e923c4a:0:becomeActiveMaster-HFileCleaner.large.0-1731680080231 {}] cleaner.HFileCleaner(306): Exit Thread[master/26754e923c4a:0:becomeActiveMaster-HFileCleaner.large.0-1731680080231,5,FailOnTimeoutGroup] 2024-11-15T14:17:36,341 DEBUG [master/26754e923c4a:0:becomeActiveMaster-HFileCleaner.small.0-1731680080232 {}] cleaner.HFileCleaner(306): Exit Thread[master/26754e923c4a:0:becomeActiveMaster-HFileCleaner.small.0-1731680080232,5,FailOnTimeoutGroup] 2024-11-15T14:17:36,342 INFO [M:0;26754e923c4a:35807 {}] hbase.ChoreService(370): Chore service for: master/26754e923c4a:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T14:17:36,342 INFO [M:0;26754e923c4a:35807 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T14:17:36,342 DEBUG [M:0;26754e923c4a:35807 {}] master.HMaster(1795): Stopping service threads 2024-11-15T14:17:36,342 INFO [M:0;26754e923c4a:35807 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T14:17:36,342 INFO [M:0;26754e923c4a:35807 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T14:17:36,342 ERROR [M:0;26754e923c4a:35807 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:39863 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:39863,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-15T14:17:36,343 INFO [M:0;26754e923c4a:35807 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T14:17:36,343 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T14:17:36,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T14:17:36,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T14:17:36,348 DEBUG [M:0;26754e923c4a:35807 {}] zookeeper.ZKUtil(347): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T14:17:36,349 WARN [M:0;26754e923c4a:35807 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T14:17:36,349 INFO [M:0;26754e923c4a:35807 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/.lastflushedseqids 2024-11-15T14:17:36,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742499_1675 (size=72) 2024-11-15T14:17:36,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T14:17:36,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40177-0x1014006c21b0001, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T14:17:36,433 INFO [RS:0;26754e923c4a:40177 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T14:17:36,434 INFO [RS:0;26754e923c4a:40177 {}] regionserver.HRegionServer(1031): Exiting; stopping=26754e923c4a,40177,1731680077530; zookeeper connection closed. 2024-11-15T14:17:36,434 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@40a20ea3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@40a20ea3 2024-11-15T14:17:36,435 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-15T14:17:36,558 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T14:17:36,558 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T14:17:36,559 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T14:17:36,762 INFO [M:0;26754e923c4a:35807 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T14:17:36,763 INFO [M:0;26754e923c4a:35807 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T14:17:36,763 DEBUG [M:0;26754e923c4a:35807 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T14:17:36,763 INFO [M:0;26754e923c4a:35807 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T14:17:36,763 DEBUG [M:0;26754e923c4a:35807 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T14:17:36,763 DEBUG [M:0;26754e923c4a:35807 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T14:17:36,763 DEBUG [M:0;26754e923c4a:35807 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T14:17:36,764 INFO [M:0;26754e923c4a:35807 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=798.78 KB heapSize=980.02 KB 2024-11-15T14:17:36,787 DEBUG [M:0;26754e923c4a:35807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4a4a38b48134ddd90f852552e038fc2 is 82, key is hbase:meta,,1/info:regioninfo/1731680081439/Put/seqid=0 2024-11-15T14:17:36,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742500_1676 (size=5672) 2024-11-15T14:17:37,191 INFO [M:0;26754e923c4a:35807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2248 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4a4a38b48134ddd90f852552e038fc2 2024-11-15T14:17:37,213 DEBUG [M:0;26754e923c4a:35807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f413dee262d243328b0f0014f656bda4 is 2480, key is \x00\x00\x00\x00\x00\x00\x00`/proc:d/1731680167699/Put/seqid=0 2024-11-15T14:17:37,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742501_1677 (size=45435) 2024-11-15T14:17:37,617 INFO [M:0;26754e923c4a:35807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=798.22 KB at sequenceid=2248 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f413dee262d243328b0f0014f656bda4 2024-11-15T14:17:37,621 INFO [M:0;26754e923c4a:35807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f413dee262d243328b0f0014f656bda4 2024-11-15T14:17:37,638 DEBUG [M:0;26754e923c4a:35807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/25f40d65d67c4e2f8386dd7d7baa2af7 is 69, key is 26754e923c4a,40177,1731680077530/rs:state/1731680080359/Put/seqid=0 2024-11-15T14:17:37,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073742502_1678 (size=5156) 2024-11-15T14:17:38,042 INFO [M:0;26754e923c4a:35807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2248 (bloomFilter=true), to=hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/25f40d65d67c4e2f8386dd7d7baa2af7 2024-11-15T14:17:38,045 DEBUG [M:0;26754e923c4a:35807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4a4a38b48134ddd90f852552e038fc2 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f4a4a38b48134ddd90f852552e038fc2 2024-11-15T14:17:38,048 INFO [M:0;26754e923c4a:35807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f4a4a38b48134ddd90f852552e038fc2, entries=8, sequenceid=2248, filesize=5.5 K 2024-11-15T14:17:38,049 DEBUG [M:0;26754e923c4a:35807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f413dee262d243328b0f0014f656bda4 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f413dee262d243328b0f0014f656bda4 2024-11-15T14:17:38,052 INFO [M:0;26754e923c4a:35807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f413dee262d243328b0f0014f656bda4 2024-11-15T14:17:38,052 INFO [M:0;26754e923c4a:35807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f413dee262d243328b0f0014f656bda4, entries=170, sequenceid=2248, filesize=44.4 K 2024-11-15T14:17:38,053 DEBUG [M:0;26754e923c4a:35807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/25f40d65d67c4e2f8386dd7d7baa2af7 as hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/25f40d65d67c4e2f8386dd7d7baa2af7 2024-11-15T14:17:38,055 INFO [M:0;26754e923c4a:35807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39863/user/jenkins/test-data/39226926-d14f-5d90-318e-b1d4167d2931/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/25f40d65d67c4e2f8386dd7d7baa2af7, entries=1, sequenceid=2248, filesize=5.0 K 2024-11-15T14:17:38,056 INFO [M:0;26754e923c4a:35807 {}] regionserver.HRegion(3140): Finished flush of dataSize ~798.78 KB/817947, heapSize ~979.72 KB/1003232, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1293ms, sequenceid=2248, compaction requested=false 2024-11-15T14:17:38,057 INFO [M:0;26754e923c4a:35807 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T14:17:38,057 DEBUG [M:0;26754e923c4a:35807 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731680256763Disabling compacts and flushes for region at 1731680256763Disabling writes for close at 1731680256763Obtaining lock to block concurrent updates at 1731680256764 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731680256764Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=817947, getHeapSize=1003472, getOffHeapSize=0, getCellsCount=2443 at 1731680256764Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731680256765 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731680256765Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731680256787 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731680256787Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731680257194 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731680257213 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731680257213Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731680257621 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731680257637 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731680257637Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40f1fc8b: reopening flushed file at 1731680258045 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d93f811: reopening flushed file at 1731680258048 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3825693e: reopening flushed file at 1731680258052 (+4 ms)Finished flush of dataSize ~798.78 KB/817947, heapSize ~979.72 KB/1003232, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1293ms, sequenceid=2248, compaction requested=false at 1731680258056 (+4 ms)Writing region close event to WAL at 1731680258057 (+1 ms)Closed at 1731680258057 2024-11-15T14:17:38,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45147 is added to blk_1073741830_1006 (size=965266) 2024-11-15T14:17:38,060 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T14:17:38,060 INFO [M:0;26754e923c4a:35807 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T14:17:38,060 INFO [M:0;26754e923c4a:35807 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35807 2024-11-15T14:17:38,060 INFO [M:0;26754e923c4a:35807 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T14:17:38,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T14:17:38,170 INFO [M:0;26754e923c4a:35807 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T14:17:38,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35807-0x1014006c21b0000, quorum=127.0.0.1:49996, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T14:17:38,174 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T14:17:38,176 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T14:17:38,176 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T14:17:38,177 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T14:17:38,177 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/hadoop.log.dir/,STOPPED} 2024-11-15T14:17:38,179 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T14:17:38,179 WARN [BP-1184182586-172.17.0.3-1731680072216 heartbeating to localhost/127.0.0.1:39863 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T14:17:38,179 WARN [BP-1184182586-172.17.0.3-1731680072216 heartbeating to localhost/127.0.0.1:39863 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1184182586-172.17.0.3-1731680072216 (Datanode Uuid 08c90d75-bb97-4e69-9a5b-97e84a0a674e) service to localhost/127.0.0.1:39863 2024-11-15T14:17:38,179 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T14:17:38,182 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/cluster_254b7fe7-4d7b-6185-a297-1a903e5d7b50/data/data1/current/BP-1184182586-172.17.0.3-1731680072216 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T14:17:38,182 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/cluster_254b7fe7-4d7b-6185-a297-1a903e5d7b50/data/data2/current/BP-1184182586-172.17.0.3-1731680072216 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T14:17:38,183 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T14:17:38,191 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T14:17:38,192 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T14:17:38,192 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T14:17:38,192 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T14:17:38,192 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40c584be-48b5-e808-4184-54bca8959084/hadoop.log.dir/,STOPPED} 2024-11-15T14:17:38,207 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T14:17:38,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down